Modify the log system: Automatically generate directories & Added parsing of the modbus protocol

configuration.py

@@ -1,6 +1,5 @@
 import json
 import os
-from logging_config import general as logger
 import sys
 from AES_crypt import decrypt
 
@@ -12,7 +11,7 @@ class Configuration:
             with open(self.path) as json_file:
                 self.config = json.load(json_file)
         except FileNotFoundError as e:
-            logger.error(f"当前路径：{os.getcwd()}无法找到配置文件:{e}")
+            print(f"当前路径：{os.getcwd()}无法找到配置文件:{e}")
             input("按任意键退出！")
             sys.exit()
 

connectors/__init__.py

@@ -1,3 +1,3 @@
 from . import tcp_connector
 from . import shuizhi_tcp_connector
-from . import modbus_rtu_over_tcp_connector
+# from . import modbus_rtu_over_tcp_connector

connectors/modbus_rtu_over_tcp_connector.py

@@ -3,8 +3,8 @@
 @Desc  : 目前此连接器用于的传感器有：insitu水质传感器，水质传感器，气象传感器
 """
 import json
-import threading
 import queue
+import threading
 import time
 from modbus_tk import modbus_rtu_over_tcp
 from connector import Connector
@@ -104,7 +104,6 @@ class ModbusRtuOverTcpConnector(Connector, threading.Thread):
             try:
                 self._master.set_timeout(3.0)  # modbus读取数据超时时间设置
                 self._master.set_verbose(True)
-                # print(device_id, ' ', function_code, " ", start_addr, " ", length)
                 receive_data = self._master.execute(device_id, function_code, start_addr, length)
                 datadict = {}
                 for i in range(len(receive_data)):
@@ -113,7 +112,9 @@ class ModbusRtuOverTcpConnector(Connector, threading.Thread):
                 result = [device_id, datadict]
                 return result
             except Exception as e:
-                logger.error(f'[{self.name}]: An error occurred while executing the read register command:{e}')
+                logger.error(f'[{self.name}]: read {device_id}, {function_code}, {start_addr}, {length}]:{repr(e)}')
+                self._reconnect()
+
         elif function_code in (5, 6, 15, 16):
             # 写寄存器
             output_value = command['output_value']
@@ -145,11 +146,15 @@ class ModbusRtuOverTcpConnector(Connector, threading.Thread):
                 except Exception as e:
                     logger.error(f"[{self.name}]: modbus_rtu,write:{e}")
             else:
-                result = self.exec_command(command=command_item)
-                format_data = None
-                if result:
-                    format_data = self.__converter.convert(self.__data_point_config, result)
-                if format_data:
-                    if format_data != "error" and format_data != 'pass':
-                        # 往redis存储数据
-                        self.__storager.real_time_data_storage(format_data)
+                try:
+                    result = self.exec_command(command=command_item)
+                    format_data = None
+                    if result:
+                        format_data = self.__converter.convert(self.__data_point_config, result)
+                    if format_data:
+                        if format_data != "error" and format_data != 'pass':
+                            # 往redis存储数据
+                            self.__storager.real_time_data_storage(format_data)
+                except Exception as e:
+                    logger.error(f'[{self.name}]: {e}')
+                    self._reconnect()

connectors/shuizhi_tcp_connector.py

@@ -3,15 +3,17 @@
 @Desc  : 原本用于insitu水质传感器的读取和解析，目前此连接器未启动，insitu读取和解析均为mudbus_rtu_over_tcp。
 """
 import binascii
+import socket
 import threading
 import time
+from binascii import *
 import struct
-import socket
+from crcmod import *
+
 from connector import Connector
 from event_storage import EventStorage
-# from logging_config import shuizhi_converter as logger
-from binascii import *
-from crcmod import *
+from logging_config import shuizhi_tcp_connector as logger
+from tools.format_value import format_value
 
 sendFlag = 0
 
@@ -152,11 +154,7 @@ class ShuizhiTcpConnector(Connector, threading.Thread):
         data = {}
         for index in self.__data_point_config:
             if index["io_point_name"] == name:
-                if index['divisor'] is not None:
-                    t = t / index['divisor']
-                if index['offset'] is not None:
-                    t = t - index['offset']
-                data = {'c' + str(index['serial_number']): t}
+                data = {'c' + str(index['serial_number']): format_value(index, t)}
                 self.__storager.real_time_data_storage(data)
 
     def SocketReceive(self, clientSocket):

connectors/tcp_connector.py

@@ -74,7 +74,7 @@ class TcpConnector(Connector, threading.Thread):
                 self.__sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                 self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                 self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)  # 在客户端开启心跳维护
-                self.__sock.settimeout(180)  # 设置超时时间3mins
+                self.__sock.settimeout(5*60)  # 设置超时时间，单位：秒
                 self.__sock.connect((self.__ip, self.__port))
                 self.__connected = True
                 logger.info(f'Reconnect to [{self.name}]:[{self.__ip}]:[{self.__port}] success !')
@@ -135,6 +135,8 @@ class TcpConnector(Connector, threading.Thread):
                     self.__storager.real_time_data_storage(format_data)
             except socket.timeout as e:
                 logger.error(f"{self.name}: {e}")
+                time.sleep(5)
+                self.__reconnect()
             except Exception as e:
                 logger.error(f"{self.name}: {e}")
                 time.sleep(5)

converters/__init__.py

@@ -4,3 +4,4 @@ from . import td266_converter
 from . import adcp_converter
 from . import cec21_converter
 from . import modbus_converter
+from . import shuizhi_converter

converters/cec21_converter.py

@@ -18,15 +18,15 @@ class CEC21Converter(Converter):
         # 格式化数据：['\x11\x11pval', '22.658', '-36.617', '11.304', '10.291', '14.330', '17.643', '82.970', '253.655', '-4.965', '-16.930', '11.813', '0\r\n']
         #                               温度，     pitch，    roll，    X流速，   Y流速，   流速，     方位，   流向，  南 - 北向流速，东 - 西向流速，电压，状态
         try:
-            logger.info(f"[{self._name}]原始接收数据: len: {len(data)}, values: {data}")
+            logger.info(f"[{self._name}]原始接收数据: values: {data}")
             data = data.decode('utf-8').split(',')
-            logger.info(f"[{self._name}]decode后数据: len: {len(data)}, values: {data}")
+            logger.info(f"[{self._name}]decode后数据: values: {data}")
             dict = {}
             for index in config:
                 name = 'c' + str(index['serial_number'])
                 i = int(index['address'])
                 dict[name] = format_value(index, data[i])
-            logger.info(f"[{self._name}]返回数据: len: {len(dict)}, values: {dict}")
+            logger.info(f"[{self._name}]返回数据: values: {dict}")
             return dict
         except Exception as e:
             logger.info(f"[{self._name}]：{repr(e)}")

converters/modbus_converter.py

@@ -7,9 +7,10 @@
 import binascii
 import math
 import struct
-from logging_config import modbus_converter as logger
+
 from converter import Converter
 from event_storage import EventStorage
+from logging_config import modbus_converter as logger
 from tools.format_value import format_value
 
 
@@ -26,8 +27,8 @@ class ModbusConverter(Converter):
         if data:
             device_id = data[0]
             data = data[1]
-            # print(device_name, data)
             format_data_dict = {}  # 列表数据转换字典数字
+            # [2, {37: 16646, 38: 30230}]
             try:
                 for index in config:
                     if int(index["device_id"]) == int(device_id):

converters/shuizhi_converter.py

@@ -0,0 +1,16 @@
+"""
+@File  : shuizhi_converter.py
+@Author: lee
+@Date  : 2022/11/18/0018 19:31:27
+@Desc  :
+"""
+from converter import Converter
+
+
+class ShuizhiConverter(Converter):
+
+    def __init__(self, name):
+        self._name = name
+
+    def convert(self, config, data):
+        pass

converters/td266_converter.py

@@ -15,7 +15,7 @@ class TD266Converter(Converter):
         # split('\t'): data = ['4420', '1194', '29.823', '104.507', '-7.471', '28.872', '253.153', '9.369', '1.816', '91.491', '-59.593', '100', '9.542', '9.589', '0.015']
         try:
             logger.info(f"[{self._name}]原始接收数据: len: {len(data)}, values: {data}")
-            data = data.decode('utf-8').split('\t')
+            data = data[:90].decode('utf-8').split('\t')
             logger.info(f"[{self._name}]解码分割: len: {len(data)}, values: {data}")
             dict = {}
             for index in config:

gateway.py

@@ -21,14 +21,14 @@ import logging.config
 
 # logging config
 logging.config.dictConfig(LOGGING_CONFIG)
-handlers = LOGGING_CONFIG['handlers']
-for handler in handlers:
-    item = handlers[handler]
-    if 'filename' in item:
-        filename = item['filename']
-        dirname = os.path.dirname(filename)
-        if not os.path.exists(dirname):
-            os.makedirs(dirname)
+# handlers = LOGGING_CONFIG['handlers']
+# for handler in handlers:
+#     item = handlers[handler]
+#     if 'filename' in item:
+#         filename = item['filename']
+#         dirname = os.path.dirname(filename)
+#         if not os.path.exists(dirname):
+#             os.makedirs(dirname)
 # --------------------------
 gateway_storage = EventStorage()
 connector_config = gateway_storage.get_connector_config()

gateway.spec

@@ -5,7 +5,7 @@ block_cipher = None
 
 
 a = Analysis(['gateway.py'],
-             pathex=['E:\\WorkSpace\\DataCollectionDevice\\venv\\Lib\\site-packages'],
+             pathex=['.\\venv\\Lib\\site-packages'],
              binaries=[],
              datas=[],
              hiddenimports=[],

hard_disk_storage.py

@@ -1,5 +1,7 @@
 import datetime
-from logging_config import general as logger
+import math
+import re
+
 import openpyxl
 import pymysql
 import traceback
@@ -24,7 +26,7 @@ class HardDiskStorage:
             self.conn = pymysql.connect(host=self.host, user=self.user, password=self.passwd, database=self.db, port=self.port, autocommit=True)
             return True
         except Exception as e:
-            logger.error(f'failed to connect to {self.host}:{self.port}:{self.db} by [{self.user}:{self.passwd}]:{e}')
+            print(f'failed to connect to {self.host}:{self.port}:{self.db} by [{self.user}:{self.passwd}]:{e}')
             return False
 
     def _reConn(self, num=28800, stime=3):  # 重试连接总次数为1天,这里根据实际情况自己设置,如果服务器宕机1天都没发现就......
@@ -162,6 +164,10 @@ class HardDiskStorage:
         sheet = book.create_sheet(index=0)
         # 循环将表头写入到sheet页
         for i in range(len(title)):
+            if search_info['deviceName'] == 'adcp':  # adcp需要重新计算垂直方向表头
+                name = re.findall(r"\d+\.?\d*", title[i])  # 提取数字字符
+                if name:
+                    title[i] = re.sub(r"\d+\.?\d*", str(round((int(name[0]) * math.sin(14 * math.pi / 180)), 2)), title[i])  # 14为adcp安装倾斜角度，2保留小数的位数
             sheet.cell(1, i + 1).value = title[i]
         # 写数据
         for row in range(0, len(res)):

logging_config.py

@@ -5,6 +5,7 @@
 @Desc  :
 """
 import logging
+import os
 import sys
 
 LOGGING_CONFIG = dict(
@@ -24,6 +25,12 @@ LOGGING_CONFIG = dict(
             "propagate": True,
             "qualname": "modbus_connector.debug",
         },
+        "shuizhi_tcp_connector": {
+            "level": "INFO",
+            "handlers": ["console", "shuizhi_tcp_connector"],
+            "propagate": True,
+            "qualname": "shuizhi_tcp_connector.debug",
+        },
         "tcp_connector": {
             "level": "INFO",
             "handlers": ["console", "tcp_connector"],
@@ -66,6 +73,12 @@ LOGGING_CONFIG = dict(
             "propagate": True,
             "qualname": "modbus_converter.debug",
         },
+        "shuizhi_converter": {
+            "level": "DEBUG",
+            "handlers": ["console", "shuizhi_converter"],
+            "propagate": True,
+            "qualname": "shuizhi_converter.debug",
+        },
     },
     handlers={
         # 数据采集程序控制台输出handler
@@ -92,6 +105,15 @@ LOGGING_CONFIG = dict(
             "backupCount": 20,
             "encoding": "utf-8"
         },
+        "shuizhi_tcp_connector": {
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': 'log/shuizhi_tcp_connector/shuizhi_tcp_connector.log',
+            'maxBytes': 10 * 1024 * 1024,
+            'delay': True,
+            "formatter": "generic",
+            "backupCount": 20,
+            "encoding": "utf-8"
+        },
         "tcp_connector": {
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': 'log/tcp_connector/tcp_connector.log',
@@ -155,6 +177,15 @@ LOGGING_CONFIG = dict(
             "backupCount": 20,
             "encoding": "utf-8"
         },
+        "shuizhi_converter": {
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': 'log/shuizhi_converter/shuizhi_converter.log',
+            'maxBytes': 10 * 1024 * 1024,
+            'delay': True,
+            "formatter": "generic",
+            "backupCount": 20,
+            "encoding": "utf-8"
+        },
     },
     formatters={
         # 自定义文件格式化器
@@ -168,9 +199,36 @@ LOGGING_CONFIG = dict(
 general = logging.getLogger("general")
 modbus_connector = logging.getLogger("modbus_connector")
 tcp_connector = logging.getLogger("tcp_connector")
-sm140_converter = logging.getLogger("sm140_converter")
+shuizhi_tcp_connector = logging.getLogger("shuizhi_tcp_connector")
+
 wxt536_converter = logging.getLogger("wxt536_converter")
+sm140_converter = logging.getLogger("sm140_converter")
+td266_converter = logging.getLogger("td266_converter")
+shuizhi_converter = logging.getLogger("shuizhi_converter")
 adcp_converter = logging.getLogger("adcp_converter")
 cec21_converter = logging.getLogger("cec21_converter")
-td266_converter = logging.getLogger("td266_converter")
 modbus_converter = logging.getLogger("modbus_converter")
+# 根据连接器、解析器生成日志目录---------------------------------------
+from event_storage import EventStorage
+
+gateway_storage = EventStorage()
+
+connector_config = gateway_storage.get_connector_config()
+
+handlers = LOGGING_CONFIG['handlers']
+modules = []
+for module in connector_config:
+    connector_module = module['connector_module']
+    converter_module = module['converter_module']
+    modules.append(converter_module)
+    modules.append(connector_module)
+for handler in handlers:
+    if handler in modules or handler == 'general' or handler == 'sm140_converter':
+        print(handler)
+        item = handlers[handler]
+        if 'filename' in item:
+            filename = item['filename']
+            dirname = os.path.dirname(filename)
+            if not os.path.exists(dirname):
+                os.makedirs(dirname)
+                print("新生成日志目录：", dirname)

memory_storage.py

@@ -15,7 +15,7 @@ class MemoryStorage:
         try:
             pipe = self.conn.pipeline(transaction=True)
             for key_name in data_dict.keys():
-                pipe.set(key_name, data_dict[key_name], ex=1800)  # redis过期时间30mines
+                pipe.set(key_name, data_dict[key_name], ex=30*60)  # redis过期时间
             pipe.execute()
         except Exception as e:
             print(e)

README.md → readme.md

@@ -1,7 +1,7 @@
 # 多参数监测终端源码仓库
 
 ## 程序打包exe命令
-pyinstaller -F -p E:\WorkSpace\DataCollectionDevice\venv\Lib\site-packages gateway.py
+pyinstaller -F -p .\venv\Lib\site-packages .\gateway.py
 ## 更新日志
 1. 第一次提交。时间：2021/07/19 15:23:21
 2. 修改一些代码bug，删除不必要注释。时间：2021/08/19 13:35:50

tools/format_value.py

@@ -1,5 +1,7 @@
 def format_value(index, value):
     """格式化数据"""
+    if value == '':
+        return ''
     if value is not None:
         value = float(value)
         divisor = index['divisor']
@@ -11,7 +13,7 @@ def format_value(index, value):
         if offset:
             value -= offset
         # 保留两位小数
-        # value = round(value, 2)
+        value = round(value, 3)
         if low_limit is not None and up_limit is not None:  # 上下限都存在
             if low_limit <= value <= up_limit:
                 return value