From 79ad7a596a1e96e05cd3dd3bd6b880e536e927e2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 6 Mar 2021 11:09:05 +0000 Subject: [PATCH 01/11] Pre-Release 4.2.0-beta1: version and changelog --- ChangeLog.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++ config.py | 2 +- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 48743c1c..74604327 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,72 @@ This is the master changelog for Elite Dangerous Market Connector. Entries are in reverse chronological order (latest first). --- +Pre-Release 4.2.0-beta1 +=== + +*NB: This contains further work on top of 4.1.7-rc1. Due to the major change +in how multiple-instance checking is done we felt the need to bump the minor +version.* + +There is a major change in this release with respect to how the main +application checks if there is already another instance running. + +For most users things will operate no differently, although note that the +multiple instance check does now apply to platforms other than Windows. + +For anyone wanting to run multiple instances of the program this is now +possible via: + +`runas /user:OTHERUSER EDMarketConnector.exe --force-localserver-for-auth` + +The old check was based solely on there being a window present with the title +we expect. This prevented using `runas /user:SOMEUSER ...` to run a second +copy of the application, as the resulting window would still be within the +same desktop environment and thus be found in the check. + +The new method does assume that the Journals directory is writable by the +user we're running as. This might not be true in the case of sharing the +file system to another host in a read-only manner. If we fail to open the +lock file read-write then the application aborts the checks and will simply +continue running as normal. + +Note that any single instance of EDMarketConnector.exe will still only monitor +and act upon the *latest* Journal file in the configured location. If you run +Elite Dangerous for another Commander then the application will want to start +monitoring that separate Commander. See [wiki:Troubleshooting#i-run-two-instances-of-ed-simultaneously-but-i-cant-run-two-instances-of-edmc](https://github.com/EDCD/EDMarketConnector/wiki/Troubleshooting#i-run-two-instances-of-ed-simultaneously-but-i-cant-run-two-instances-of-edmc>) +which will be updated when this change is in a full release. + +* Changes the "is there another process already running?" check to be based on + a lockfile in the configured Journals directory. The name of this file is + `edmc-journal-lock.txt` and upon successful locking it will contain text + like: + + ``` + Path: + PID: + ``` + The lock will be released and applied to the new directory if you change it + via Settings > Configuration. If the new location is already locked you'll + get a 'Retry/Ignore?' pop-up. + +* Adds the command-line argument `--force-localserver-for-auth`. This forces + using a local webserver for the Frontier Auth callback. This should be used + when running multiple instances of the application **for all instances** + else there's no guarantee of the `edmc://` protocol callback reaching the + correct process and Frontier Auth will fail. + +* Adds Steam and Epic to the list of "audiences" in the Frontier Auth callout + so that you can authorise using those accounts, rather than their associated + Frontier Account details. + +* Adds the command-line argument `--suppress-dupe-process-popup` to exit + without showing the warning popup in the case that EDMarketConnector found + another process already running. + + This can be useful if wanting to blindly run both EDMC and the game from a + batch file or similar. + + Release 4.1.6 === diff --git a/config.py b/config.py index d9ac1099..eacde870 100644 --- a/config.py +++ b/config.py @@ -13,7 +13,7 @@ appcmdname = 'EDMC' # appversion **MUST** follow Semantic Versioning rules: # # Major.Minor.Patch(-prerelease)(+buildmetadata) -appversion = '4.1.6' #-rc1+a872b5f' +appversion = '4.2.0-beta1' #-rc1+a872b5f' # For some things we want appversion without (possible) +build metadata appversion_nobuild = str(semantic_version.Version(appversion).truncate('prerelease')) copyright = u'© 2015-2019 Jonathan Harris, 2020 EDCD' From 5f3bd9be7189f85443995223f2102566db5bc06a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 5 Mar 2021 16:27:23 +0000 Subject: [PATCH 02/11] Add Steam and EGS to Frontier Auth audiences. --- companion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/companion.py b/companion.py index 1be9d0fa..b75d39ae 100644 --- a/companion.py +++ b/companion.py @@ -277,7 +277,7 @@ class Auth(object): logger.info(f'Trying auth from scratch for Commander "{self.cmdr}"') challenge = self.base64_url_encode(hashlib.sha256(self.verifier).digest()) webbrowser.open( - f'{SERVER_AUTH}{URL_AUTH}?response_type=code&audience=frontier&scope=capi&client_id={CLIENT_ID}&code_challenge={challenge}&code_challenge_method=S256&state={self.state}&redirect_uri={protocolhandler.redirect}' # noqa: E501 # I cant make this any shorter + f'{SERVER_AUTH}{URL_AUTH}?response_type=code&audience=frontier,steam,epic&scope=capi&client_id={CLIENT_ID}&code_challenge={challenge}&code_challenge_method=S256&state={self.state}&redirect_uri={protocolhandler.redirect}' # noqa: E501 # I cant make this any shorter ) return None From 67a91cf7d13042716cbd9a05550d78820381a44f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 5 Mar 2021 16:57:02 +0000 Subject: [PATCH 03/11] CAPI: Handle when we get no 'commander' in returned data. I was testing the new Steam or Epic CAPI auth. My EGS account hasn't yet been used, so has no commander attached. EDMC thinks the auth has succeeded in this case, but hitting 'Update' causes it to error because the returned data is empty. So, add some checks for lack of 'commander' key and a specific message "CAPI: No commander data returned" for status line. Without this there's a KeyError exception thrown, causing the status line to just get 'commander' in it, which isn't helpful. --- EDMarketConnector.py | 10 +++++++++- L10n/en.template | 3 +++ companion.py | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 73f15e67..6f4b8c7c 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -662,17 +662,25 @@ class AppWindow(object): config.set('querytime', querytime) # Validation - if not data.get('commander', {}).get('name'): + if 'commander' not in data: + # This can happen with EGS Auth if no commander created yet + self.status['text'] = _('CAPI: No commander data returned') + + elif not data.get('commander', {}).get('name'): self.status['text'] = _("Who are you?!") # Shouldn't happen + elif (not data.get('lastSystem', {}).get('name') or (data['commander'].get('docked') and not data.get('lastStarport', {}).get('name'))): # Only care if docked self.status['text'] = _("Where are you?!") # Shouldn't happen + elif not data.get('ship', {}).get('name') or not data.get('ship', {}).get('modules'): self.status['text'] = _("What are you flying?!") # Shouldn't happen + elif monitor.cmdr and data['commander']['name'] != monitor.cmdr: # Companion API return doesn't match Journal raise companion.CmdrError() + elif ((auto_update and not data['commander'].get('docked')) or (data['lastSystem']['name'] != monitor.system) or ((data['commander']['docked'] diff --git a/L10n/en.template b/L10n/en.template index 23b4abcd..ef8f7399 100644 --- a/L10n/en.template +++ b/L10n/en.template @@ -46,6 +46,9 @@ /* Folder selection button on Windows. [prefs.py] */ "Browse..." = "Browse..."; +/* No 'commander' data in CAPI [EDMarketConnector.py] */ +"CAPI: No commander data returned" = "CAPI: No commander data returned"; + /* Federation rank. [stats.py] */ "Cadet" = "Cadet"; diff --git a/companion.py b/companion.py index b75d39ae..ba8faf1a 100644 --- a/companion.py +++ b/companion.py @@ -532,6 +532,10 @@ class Session(object): def station(self) -> CAPIData: """Perform CAPI /profile endpoint query for station data.""" data = self.query(URL_QUERY) + if 'commander' not in data: + logger.error('No commander in returned data') + return data + if not data['commander'].get('docked'): return data From 71858357eb639b65521cef2122a4e82caa642ebf Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 6 Mar 2021 10:54:15 +0000 Subject: [PATCH 04/11] CAPI: Log if no commander in profile() data --- companion.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/companion.py b/companion.py index ba8faf1a..850f3a7a 100644 --- a/companion.py +++ b/companion.py @@ -527,7 +527,11 @@ class Session(object): def profile(self) -> CAPIData: """Perform general CAPI /profile endpoint query.""" - return self.query(URL_QUERY) + data = self.query(URL_QUERY) + if 'commander' not in data: + logger.error('No commander in returned data') + + return data def station(self) -> CAPIData: """Perform CAPI /profile endpoint query for station data.""" From 80529986586b763e99e56245a180bdd398faa6a2 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 6 Mar 2021 18:55:35 +0000 Subject: [PATCH 05/11] Pre-Release 4.2.0-beta2: version and changelog --- ChangeLog.md | 6 ++++++ config.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 74604327..c2dfc3e1 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,12 @@ This is the master changelog for Elite Dangerous Market Connector. Entries are in reverse chronological order (latest first). --- +Pre-Release 4.2.0-beta2 +=== + +This release actually includes the commits to enable Steam and Epic +authentication. + Pre-Release 4.2.0-beta1 === diff --git a/config.py b/config.py index eacde870..23dbfc4b 100644 --- a/config.py +++ b/config.py @@ -13,7 +13,7 @@ appcmdname = 'EDMC' # appversion **MUST** follow Semantic Versioning rules: # # Major.Minor.Patch(-prerelease)(+buildmetadata) -appversion = '4.2.0-beta1' #-rc1+a872b5f' +appversion = '4.2.0-beta2' #-rc1+a872b5f' # For some things we want appversion without (possible) +build metadata appversion_nobuild = str(semantic_version.Version(appversion).truncate('prerelease')) copyright = u'© 2015-2019 Jonathan Harris, 2020 EDCD' From c461f5d45bb235b131a4daf3ac3ebe364eab5e25 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 6 Mar 2021 19:07:04 +0000 Subject: [PATCH 06/11] 4.2.0-beta2: changelog: Attempt to cite actually tested runas command --- ChangeLog.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index c2dfc3e1..eaf4383e 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -7,6 +7,17 @@ Pre-Release 4.2.0-beta2 This release actually includes the commits to enable Steam and Epic authentication. +**NB: The correct form for the runas command is as follows:** + +`runas /user: "\\"c:\\Program Files (x86)\\EDMarketConnector\\EDMarketConnector.exe\\" --force-localserver-for-auth"` + +If anything has messed with the backslash characters there then know that you +need to have " (double-quote) around the entire command (path to program .exe +*and* any extra arguments), and as a result need to place a backslash before +any double-quote characters in the command (such as around the space-including +path to the program). + + Pre-Release 4.2.0-beta1 === From 9d816eaa79feb11f7c4b4104d884f7a6c064282f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 6 Mar 2021 19:09:41 +0000 Subject: [PATCH 07/11] un-double the blackslashes in changelog --- ChangeLog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index eaf4383e..f2727bbc 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -9,7 +9,7 @@ authentication. **NB: The correct form for the runas command is as follows:** -`runas /user: "\\"c:\\Program Files (x86)\\EDMarketConnector\\EDMarketConnector.exe\\" --force-localserver-for-auth"` +`runas /user: "\"c:\Program Files (x86)\EDMarketConnector\EDMarketConnector.exe\" --force-localserver-for-auth"` If anything has messed with the backslash characters there then know that you need to have " (double-quote) around the entire command (path to program .exe From 875ccd08f59c85902021993484227516e512c1ee Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 6 Mar 2021 19:12:12 +0000 Subject: [PATCH 08/11] Document that GitHub renders runas command correctly --- ChangeLog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog.md b/ChangeLog.md index f2727bbc..360e9e91 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -17,6 +17,7 @@ need to have " (double-quote) around the entire command (path to program .exe any double-quote characters in the command (such as around the space-including path to the program). +I've verified it renders correctly (on GitHub)[https://github.com/EDCD/EDMarketConnector/blob/Release/4.2.0-beta2/ChangeLog.md]. Pre-Release 4.2.0-beta1 === From 038d16671c08a3413409940af6583f61380d07d0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 6 Mar 2021 19:16:56 +0000 Subject: [PATCH 09/11] Correct URL format --- ChangeLog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 360e9e91..750748e5 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -17,7 +17,7 @@ need to have " (double-quote) around the entire command (path to program .exe any double-quote characters in the command (such as around the space-including path to the program). -I've verified it renders correctly (on GitHub)[https://github.com/EDCD/EDMarketConnector/blob/Release/4.2.0-beta2/ChangeLog.md]. +I've verified it renders correctly [on GitHub](https://github.com/EDCD/EDMarketConnector/blob/Release/4.2.0-beta2/ChangeLog.md). Pre-Release 4.2.0-beta1 === From a6939c77d9e67f6f5d22f8dc5837d9b6335e2def Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 9 Mar 2021 12:37:59 +0000 Subject: [PATCH 10/11] Backport of fix/891/force-localserver-for-auth_fix-journallock-import * Move JournalLock into its own file. --- EDMarketConnector.py | 2 +- monitor.py | 198 +------------------------------------------ 2 files changed, 2 insertions(+), 198 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 6f4b8c7c..bcc64c80 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -33,7 +33,7 @@ if __name__ == '__main__': # After the redirect in case config does logging setup from config import appversion, appversion_nobuild, config, copyright from EDMCLogging import edmclogger, logger, logging -from monitor import JournalLock +from journal_lock import JournalLock if __name__ == '__main__': # noqa: C901 # Command-line arguments diff --git a/monitor.py b/monitor.py index 6d4a3b88..19d35aae 100644 --- a/monitor.py +++ b/monitor.py @@ -2,16 +2,12 @@ from collections import defaultdict, OrderedDict import json import re import threading -from os import getpid as os_getpid from os import listdir, SEEK_SET, SEEK_END from os.path import basename, expanduser, isdir, join -import pathlib from sys import platform -import tkinter as tk -from tkinter import ttk from time import gmtime, localtime, sleep, strftime, strptime, time from calendar import timegm -from typing import Any, Callable, List, MutableMapping, Optional, OrderedDict as OrderedDictT, Tuple, TYPE_CHECKING, Union +from typing import Any, List, MutableMapping, Optional, OrderedDict as OrderedDictT, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: import tkinter @@ -1029,195 +1025,3 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below # singleton monitor = EDLogs() - - -class JournalLock: - """Handle locking of journal directory.""" - - def __init__(self): - """Initialise where the journal directory and lock file are.""" - self.journal_dir: str = config.get('journaldir') or config.default_journal_dir - self.journal_dir_path = pathlib.Path(self.journal_dir) - self.journal_dir_lockfile_name = None - self.journal_dir_lockfile = None - - def obtain_lock(self) -> bool: - """ - Attempt to obtain a lock on the journal directory. - - :return: bool - True if we successfully obtained the lock - """ - - self.journal_dir_lockfile_name = self.journal_dir_path / 'edmc-journal-lock.txt' - logger.trace(f'journal_dir_lockfile_name = {self.journal_dir_lockfile_name!r}') - try: - self.journal_dir_lockfile = open(self.journal_dir_lockfile_name, mode='w+', encoding='utf-8') - - # Linux CIFS read-only mount throws: OSError(30, 'Read-only file system') - # Linux no-write-perm directory throws: PermissionError(13, 'Permission denied') - except Exception as e: # For remote FS this could be any of a wide range of exceptions - logger.warning(f"Couldn't open \"{self.journal_dir_lockfile_name}\" for \"w+\"" - f" Aborting duplicate process checks: {e!r}") - - if platform == 'win32': - logger.trace('win32, using msvcrt') - # win32 doesn't have fcntl, so we have to use msvcrt - import msvcrt - - try: - msvcrt.locking(self.journal_dir_lockfile.fileno(), msvcrt.LK_NBLCK, 4096) - - except Exception as e: - logger.info(f"Exception: Couldn't lock journal directory \"{self.journal_dir}\"" - f", assuming another process running: {e!r}") - return False - - else: - logger.trace('NOT win32, using fcntl') - try: - import fcntl - - except ImportError: - logger.warning("Not on win32 and we have no fcntl, can't use a file lock!" - "Allowing multiple instances!") - return True # Lie about being locked - - try: - fcntl.flock(self.journal_dir_lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - - except Exception as e: - logger.info(f"Exception: Couldn't lock journal directory \"{self.journal_dir}\", " - f"assuming another process running: {e!r}") - return False - - self.journal_dir_lockfile.write(f"Path: {self.journal_dir}\nPID: {os_getpid()}\n") - self.journal_dir_lockfile.flush() - - logger.trace('Done') - return True - - def release_lock(self) -> bool: - """ - Release lock on journal directory. - - :return: bool - Success of unlocking operation.""" - unlocked = False - if platform == 'win32': - logger.trace('win32, using msvcrt') - # win32 doesn't have fcntl, so we have to use msvcrt - import msvcrt - - try: - # Need to seek to the start first, as lock range is relative to - # current position - self.journal_dir_lockfile.seek(0) - msvcrt.locking(self.journal_dir_lockfile.fileno(), msvcrt.LK_UNLCK, 4096) - - except Exception as e: - logger.info(f"Exception: Couldn't unlock journal directory \"{self.journal_dir}\": {e!r}") - - else: - unlocked = True - - else: - logger.trace('NOT win32, using fcntl') - try: - import fcntl - - except ImportError: - logger.warning("Not on win32 and we have no fcntl, can't use a file lock!") - return True # Lie about being unlocked - - try: - fcntl.flock(self.journal_dir_lockfile, fcntl.LOCK_UN) - - except Exception as e: - logger.info(f"Exception: Couldn't unlock journal directory \"{self.journal_dir}\": {e!r}") - - else: - unlocked = True - - # Close the file whether or not the unlocking succeeded. - self.journal_dir_lockfile.close() - - self.journal_dir_lockfile_name = None - self.journal_dir_lockfile = None - - return unlocked - - class JournalAlreadyLocked(tk.Toplevel): - """Pop-up for when Journal directory already locked.""" - - def __init__(self, parent: tk.Tk, callback: Callable): - tk.Toplevel.__init__(self, parent) - - self.parent = parent - self.callback = callback - self.title(_('Journal directory already locked')) - - # remove decoration - if platform == 'win32': - self.attributes('-toolwindow', tk.TRUE) - - elif platform == 'darwin': - # http://wiki.tcl.tk/13428 - parent.call('tk::unsupported::MacWindowStyle', 'style', self, 'utility') - - self.resizable(tk.FALSE, tk.FALSE) - - frame = ttk.Frame(self) - frame.grid(sticky=tk.NSEW) - - self.blurb = tk.Label(frame) - self.blurb['text'] = _("The new Journal Directory location is already locked.{CR}" - "You can either attempt to resolve this and then Retry, or choose to Ignore this.") - self.blurb.grid(row=1, column=0, columnspan=2, sticky=tk.NSEW) - - self.retry_button = ttk.Button(frame, text=_('Retry'), command=self.retry) - self.retry_button.grid(row=2, column=0, sticky=tk.EW) - - self.ignore_button = ttk.Button(frame, text=_('Ignore'), command=self.ignore) - self.ignore_button.grid(row=2, column=1, sticky=tk.EW) - self.protocol("WM_DELETE_WINDOW", self._destroy) - - def retry(self): - logger.trace('User selected: Retry') - self.destroy() - self.callback(True, self.parent) - - def ignore(self): - logger.trace('User selected: Ignore') - self.destroy() - self.callback(False, self.parent) - - def _destroy(self): - logger.trace('User force-closed popup, treating as Ignore') - self.ignore() - - def update_lock(self, parent: tk.Tk): - """Update journal directory lock to new location if possible.""" - current_journaldir = config.get('journaldir') or config.default_journal_dir - - if current_journaldir == self.journal_dir: - return # Still the same - - self.release_lock() - - self.journal_dir = current_journaldir - self.journal_dir_path = pathlib.Path(self.journal_dir) - if not self.obtain_lock(): - # Pop-up message asking for Retry or Ignore - self.retry_popup = self.JournalAlreadyLocked(parent, self.retry_lock) - - def retry_lock(self, retry: bool, parent: tk.Tk): - logger.trace(f'We should retry: {retry}') - - if not retry: - return - - current_journaldir = config.get('journaldir') or config.default_journal_dir - self.journal_dir = current_journaldir - self.journal_dir_path = pathlib.Path(self.journal_dir) - if not self.obtain_lock(): - # Pop-up message asking for Retry or Ignore - self.retry_popup = self.JournalAlreadyLocked(parent, self.retry_lock) From 773629afbe9b836dbdc1a22faee2c7e8bc3fe3c5 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 9 Mar 2021 12:43:48 +0000 Subject: [PATCH 11/11] Pre-Release 4.2.0-beta3: version and changelog --- ChangeLog.md | 7 +++++++ config.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ChangeLog.md b/ChangeLog.md index 750748e5..35c6cd5c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,13 @@ This is the master changelog for Elite Dangerous Market Connector. Entries are in reverse chronological order (latest first). --- +Pre-Release 4.2.0-beta3 +=== + +* Allow `--force-localserver-for-auth` to actually work. See [#891 + `--force-localserver-for-auth` fails due to JounalLock being imported from + monitor.py](https://github.com/EDCD/EDMarketConnector/issues/891) + Pre-Release 4.2.0-beta2 === diff --git a/config.py b/config.py index 23dbfc4b..bf9d4fee 100644 --- a/config.py +++ b/config.py @@ -13,7 +13,7 @@ appcmdname = 'EDMC' # appversion **MUST** follow Semantic Versioning rules: # # Major.Minor.Patch(-prerelease)(+buildmetadata) -appversion = '4.2.0-beta2' #-rc1+a872b5f' +appversion = '4.2.0-beta3' #-rc1+a872b5f' # For some things we want appversion without (possible) +build metadata appversion_nobuild = str(semantic_version.Version(appversion).truncate('prerelease')) copyright = u'© 2015-2019 Jonathan Harris, 2020 EDCD'