使用python將日志文件轉換為json文件

Question

我是python的新手。 我正在嘗試使用python腳本將日志文件轉換為json文件。 我創建了一個主文件和一個del6文件。 在這里，它將轉換日志文件並將其寫入新的json文件。 執行時，它顯示以下錯誤。

Traceback (most recent call last):
  File "main.py", line 23, in <module>
    main()
  File "main.py", line 14, in main
    print toJson(sys.argv[2])
  File "/home/paulsteven/BEAT/apache/del6.py", line 46, in toJson
    entries = readfile(file)
  File "/home/paulsteven/BEAT/apache/del6.py", line 21, in readfile
    filecontent[index] = line2dict(line)
  File "/home/paulsteven/BEAT/apache/del6.py", line 39, in line2dict
    res = m.groupdict()
AttributeError: 'NoneType' object has no attribute 'groupdict'

我嘗試了將此鏈接日志鏈接到json，但沒有給我適當的解決方案。 有什么辦法可以解決這個問題。

這是我的示例日志文件：

February 14 2019, 15:38:47      172.217.160.132     www.google.com      up      tcp-tcp@ www.google.com     172.217.160.132
February 14 2019, 15:38:47      104.28.4.86     www.smackcoders.com     up      tcp-tcp@ www.smackcoders.com        104.28.4.86     

輸出應類似於：

{"1": {"timestamp": "February 14 2019, 15:38:47", "monitorip": "172.217.160.132 ", "monitorhost": "www.google.com", "monitorstatus": "up", "monitorid": "tcp-tcp@ www.google.com", "resolveip": "172.217.160.132"}, "2": {"timestamp": "February 14 2019, 15:38:47", "monitorip": "104.28.4.86", "monitorhost": "www.smackcoders.com", "monitorstatus": "up", "monitorid": "tcp-tcp@ www.smackcoders.com", "resolveip": "104.28.4.86"}

這是主要的python代碼：

import sys
from del6 import *

def main():
    if len(sys.argv) < 3:
        print "Incorrect Syntax. Usage: python main.py -f <filename>"
        sys.exit(2)
    elif sys.argv[1] != "-f":
        print "Invalid switch '"+sys.argv[1]+"'"
        sys.exit(2)
    elif os.path.isfile(sys.argv[2]) == False:
        print "File does not exist"
        sys.exit(2)
    print toJson(sys.argv[2])
    text_file = open("tcp.json", "a+")
    text_file.write(toJson(sys.argv[2]))
    text_file.write("\n")
    text_file.close()



if __name__ == "__main__":
    main()

這是我的del6代碼：

import fileinput
import re
import os
try: import simplejson as json
except ImportError: import json

#read input file and return entries' Dict Object
def readfile(file):
    filecontent = {}
    index = 0
    #check necessary file size checking
    statinfo = os.stat(file)

    #just a guestimate. I believe a single entry contains atleast 150 chars
    if statinfo.st_size < 150:
        print "Not a valid access_log file. It does not have enough data"
    else:
        for line in fileinput.input(file):
            index = index+1
            if line != "\n": #don't read newlines
                filecontent[index] = line2dict(line)

        fileinput.close()
    return filecontent

#gets a line of string from Log and convert it into Dict Object
def line2dict(line):
    #Snippet, thanks to http://www.seehuhn.de/blog/52
    parts = [
    r'(?P<timestamp>\S+)',                  
    r'(?P<monitorip>\S+)',               
    r'(?P<monitorhost>\S+)',                
    r'(?P<monitorstatus>\S+)',              
    r'"(?P<monitorid>\S+)"',              
    r'(?P<resolveip>\S+)',             
]
    pattern = re.compile(r'\s+'.join(parts)+r'\s*\Z')
    m = pattern.match(line)
    res = m.groupdict()
    return res

#to get jSon of entire Log
#returns JSON object
def toJson(file):
    #get dict object for each entry
    entries = readfile(file)
    return json.JSONEncoder().encode(entries)

Answer 1

我看到那列除以雙標簽。 因此基於：

i = 1
result = {}
with open('log.txt') as f:
    lines = f.readlines()
    for line in lines:
        r = line.split('\t\t')
        result[i] = {'timestamp': r[0], 'monitorip': r[1], 'monitorhost': r[2], 'monitorstatus': r[3], 'monitorid': r[4], 'resolveip': r[5]}
        i += 1

輸出：

{1: {'timestamp': 'February 14 2019, 15:38:47', 'monitorip': '172.217.160.132', 'monitorhost': 'www.google.com', 'monitorstatus': 'up', 'monitorid': 'tcp-tcp@ www.google.com', 'resolveip': '172.217.160.132\n'}, 2: {'timestamp': 'February 14 2019, 15:38:47', 'monitorip': '104.28.4.86', 'monitorhost': 'www.smackcoders.com', 'monitorstatus': 'up', 'monitorid': 'tcp-tcp@ www.smackcoders.com', 'resolveip': '104.28.4.86'}}

或者，如果您想擁有更自然的字典列表，則：

result = []
with open('log.txt') as f:
    lines = f.readlines()
    for line in lines:
        r = line.split('\t\t')
        result.append({'timestamp': r[0], 'monitorip': r[1], 'monitorhost': r[2], 'monitorstatus': r[3], 'monitorid': r[4], 'resolveip': r[5]})

輸出：

[{'timestamp': 'February 14 2019, 15:38:47', 'monitorip': '172.217.160.132', 'monitorhost': 'www.google.com', 'monitorstatus': 'up', 'monitorid': 'tcp-tcp@ www.google.com', 'resolveip': '172.217.160.132\n'}, {'timestamp': 'February 14 2019, 15:38:47', 'monitorip': '104.28.4.86', 'monitorhost': 'www.smackcoders.com', 'monitorstatus': 'up', 'monitorid': 'tcp-tcp@ www.smackcoders.com', 'resolveip': '104.28.4.86'}]

Answer 2

感謝您的回答。 要將其保存在JSON文件中：

import json


i = 1
result = {}
with open('tcp.log') as f:
    lines = f.readlines()
    for line in lines:
        r = line.split('\t\t')
        result[i] = {'timestamp': r[0], 'monitorip': r[1], 'monitorhost': r[2], 'monitorstatus': r[3], 'monitorid': r[4], 'resolveip': r[5]}
        i += 1 
print(result) 
with open('data.json', 'w') as fp:
    json.dump(result, fp)

Answer 3

下面是解決該問題的通用方法。函數'log_lines_to_json'將處理任何文本文件，其中字段由'field_delimiter'分隔並且字段名稱為'field_names'

FIELD_NAMES = ['timestamp', 'monitorip', 'monitorhost', 'monitorstatus', 'monitorid', 'resolveip']
FIELD_DELIMITER = '\t\t'


def log_lines_to_json(log_file, field_names, field_delimiter):
    result = []
    with open(log_file) as f:
        lines = f.readlines()
        for line in lines:
            fields = line.split(field_delimiter)
            result.append({field_name: fields[idx] for idx, field_name in enumerate(field_names)})
    return result


entries = log_lines_to_json('log.txt', FIELD_NAMES, FIELD_DELIMITER)
for entry in entries:
    print(entry)

輸出：

{'monitorid': 'tcp-tcp@ www.google.com', 'monitorstatus': 'up', 'timestamp': 'February 14 2019, 15:38:47', 'monitorhost': 'www.google.com', 'monitorip': '172.217.160.132', 'resolveip': '172.217.160.132\n'}
{'monitorid': 'tcp-tcp@ www.smackcoders.com', 'monitorstatus': 'up', 'timestamp': 'February 14 2019, 15:38:47', 'monitorhost': 'www.smackcoders.com', 'monitorip': '104.28.4.86', 'resolveip': '104.28.4.86'}