ED/EDMarketConnector
1
0
Fork 0
mirror of https://github.com/EDCD/EDMarketConnector.git synced 2025-06-15 06:42:15 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #781 from A-UNDERSCORE-D/fix/779-utf8-file-encodings

Browse Source 
Always read journal with utf-8 in EDMC.py
This commit is contained in:
Athanasius 2020-12-02 14:22:45 +00:00 committed by GitHub
commit 128f22941d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
EDMC.py
View File

@ -12,16 +12,20 @@ from os.path import getmtime, join
from time import sleep, time from time import sleep, time
from typing import TYPE_CHECKING, Any, Optional from typing import TYPE_CHECKING, Any, Optional
# See EDMCLogging.py docs.
# isort: off # isort: off
os.environ["EDMC_NO_UI"] = "1"
# See EDMCLogging.py docs.
# workaround for https://github.com/EDCD/EDMarketConnector/issues/568
from EDMCLogging import edmclogger, logger, logging from EDMCLogging import edmclogger, logger, logging
if TYPE_CHECKING: if TYPE_CHECKING:
from logging import trace, TRACE # type: ignore # noqa: F401 from logging import trace, TRACE # type: ignore # noqa: F401
# isort: on
edmclogger.set_channels_loglevel(logging.INFO) edmclogger.set_channels_loglevel(logging.INFO)
# workaround for https://github.com/EDCD/EDMarketConnector/issues/568 # isort: on
os.environ["EDMC_NO_UI"] = "1"
import collate import collate
import commodity import commodity
@ -169,7 +173,16 @@ sys.path: {sys.path}'''
if args.j: if args.j:
logger.debug('Import and collate from JSON dump') logger.debug('Import and collate from JSON dump')
# Import and collate from JSON dump # Import and collate from JSON dump
#
# Try twice, once with the system locale and once enforcing utf-8. If the file was made on the current
# system, chances are its the current locale, and not utf-8. Otherwise if it was copied, its probably
# utf8. Either way, try the system FIRST because reading something like cp1251 in UTF-8 results in garbage
# but the reverse results in an exception.
try:
data = json.load(open(args.j)) data = json.load(open(args.j))
except UnicodeDecodeError:
data = json.load(open(args.j, encoding='utf-8'))
config.set('querytime', int(getmtime(args.j))) config.set('querytime', int(getmtime(args.j)))
else: else:
@ -184,7 +197,7 @@ sys.path: {sys.path}'''
logfile = join(logdir, logfiles[-1]) logfile = join(logdir, logfiles[-1])
logger.debug(f'Using logfile "{logfile}"') logger.debug(f'Using logfile "{logfile}"')
with open(logfile, 'r') as loghandle: with open(logfile, 'r', encoding='utf-8') as loghandle:
for line in loghandle: for line in loghandle:
try: try:
monitor.parse_entry(line) monitor.parse_entry(line)