From 5aa7b98cf6324b1ff2fe9c4f4c3f42914a634b25 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 1 Feb 2022 14:14:43 +0000 Subject: [PATCH 1/3] setup.py: Add `util/` to win32 packages Currently only `plugins/eddn.py` imports anything from `util/`, and as the core plugins are dynamically loaded py2exe can't possibly know about this. So we need to specify it explictly. Tested with build and install, and also manually checking the resulting library.zip: ``` 0$ unzip -l library.zip | grep 'util/' 10582 2022-02-01 14:12 urllib3/util/url.pyc 11221 2022-02-01 14:12 urllib3/util/ssl_.pyc 3367 2022-02-01 14:12 urllib3/util/connection.pyc 1031 2022-02-01 14:12 urllib3/util/__init__.pyc 1266 2022-02-01 14:12 urllib3/util/proxy.pyc 3394 2022-02-01 14:12 urllib3/util/request.pyc 8863 2022-02-01 14:12 urllib3/util/timeout.pyc 986 2022-02-01 14:12 urllib3/util/queue.pyc 7332 2022-02-01 14:12 urllib3/util/ssltransport.pyc 1105 2022-02-01 14:12 util/text.pyc 3015 2022-02-01 14:12 urllib3/util/wait.pyc 110 2022-02-01 14:12 util/__init__.pyc 15704 2022-02-01 14:12 urllib3/util/retry.pyc 2279 2022-02-01 14:12 urllib3/util/response.pyc ``` --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f73dc370..57383fe7 100755 --- a/setup.py +++ b/setup.py @@ -192,6 +192,7 @@ elif sys.platform == 'win32': 'optimize': 2, 'packages': [ 'sqlite3', # Included for plugins + 'util', # 2022-02-01 only imported in plugins/eddn.py ], 'includes': [ 'dataclasses', From ac7cfb9b147fd0e90aa7ffeff81ef133f5d74399 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 1 Feb 2022 14:45:30 +0000 Subject: [PATCH 2/3] core plugins: Add big obvious comment about imports and windows installer --- plugins/coriolis.py | 22 ++++++++++++++++++++++ plugins/eddb.py | 22 ++++++++++++++++++++++ plugins/eddn.py | 22 ++++++++++++++++++++++ plugins/edsm.py | 22 ++++++++++++++++++++++ plugins/edsy.py | 22 ++++++++++++++++++++++ plugins/inara.py | 22 ++++++++++++++++++++++ 6 files changed, 132 insertions(+) diff --git a/plugins/coriolis.py b/plugins/coriolis.py index c05ba089..78ae1b41 100644 --- a/plugins/coriolis.py +++ b/plugins/coriolis.py @@ -1,5 +1,27 @@ """Coriolis ship export.""" +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# +# This is an EDMC 'core' plugin. +# +# All EDMC plugins are *dynamically* loaded at run-time. +# +# We build for Windows using `py2exe`. +# +# `py2exe` can't possibly know about anything in the dynamically loaded +# core plugins. +# +# Thus you **MUST** check if any imports you add in this file are only +# referenced in this file (or only in any other core plugin), and if so... +# +# YOU MUST ENSURE THAT PERTINENT ADJUSTMENTS ARE MADE IN `setup.py` +# SO AS TO ENSURE THE FILES ARE ACTUALLY PRESENT IN AN END-USER +# INSTALLATION ON WINDOWS. +# +# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# import base64 import gzip import io diff --git a/plugins/eddb.py b/plugins/eddb.py index 220046f0..111d768c 100644 --- a/plugins/eddb.py +++ b/plugins/eddb.py @@ -23,6 +23,28 @@ # +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# +# This is an EDMC 'core' plugin. +# +# All EDMC plugins are *dynamically* loaded at run-time. +# +# We build for Windows using `py2exe`. +# +# `py2exe` can't possibly know about anything in the dynamically loaded +# core plugins. +# +# Thus you **MUST** check if any imports you add in this file are only +# referenced in this file (or only in any other core plugin), and if so... +# +# YOU MUST ENSURE THAT PERTINENT ADJUSTMENTS ARE MADE IN `setup.py` +# SO AS TO ENSURE THE FILES ARE ACTUALLY PRESENT IN AN END-USER +# INSTALLATION ON WINDOWS. +# +# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# import sys from typing import TYPE_CHECKING, Any, Optional diff --git a/plugins/eddn.py b/plugins/eddn.py index e3e4e198..d58eac26 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -1,5 +1,27 @@ """Handle exporting data to EDDN.""" +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# +# This is an EDMC 'core' plugin. +# +# All EDMC plugins are *dynamically* loaded at run-time. +# +# We build for Windows using `py2exe`. +# +# `py2exe` can't possibly know about anything in the dynamically loaded +# core plugins. +# +# Thus you **MUST** check if any imports you add in this file are only +# referenced in this file (or only in any other core plugin), and if so... +# +# YOU MUST ENSURE THAT PERTINENT ADJUSTMENTS ARE MADE IN `setup.py` +# SO AS TO ENSURE THE FILES ARE ACTUALLY PRESENT IN AN END-USER +# INSTALLATION ON WINDOWS. +# +# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# import itertools import json import pathlib diff --git a/plugins/edsm.py b/plugins/edsm.py index 31f5a317..6bbf0223 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -9,6 +9,28 @@ # 4) Ensure the EDSM API call(back) for setting the image at end of system # text is always fired. i.e. CAPI cmdr_data() processing. +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# +# This is an EDMC 'core' plugin. +# +# All EDMC plugins are *dynamically* loaded at run-time. +# +# We build for Windows using `py2exe`. +# +# `py2exe` can't possibly know about anything in the dynamically loaded +# core plugins. +# +# Thus you **MUST** check if any imports you add in this file are only +# referenced in this file (or only in any other core plugin), and if so... +# +# YOU MUST ENSURE THAT PERTINENT ADJUSTMENTS ARE MADE IN `setup.py` +# SO AS TO ENSURE THE FILES ARE ACTUALLY PRESENT IN AN END-USER +# INSTALLATION ON WINDOWS. +# +# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# import json import threading import tkinter as tk diff --git a/plugins/edsy.py b/plugins/edsy.py index 7da92fa4..df61683e 100644 --- a/plugins/edsy.py +++ b/plugins/edsy.py @@ -1,5 +1,27 @@ # EDShipyard ship export +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# +# This is an EDMC 'core' plugin. +# +# All EDMC plugins are *dynamically* loaded at run-time. +# +# We build for Windows using `py2exe`. +# +# `py2exe` can't possibly know about anything in the dynamically loaded +# core plugins. +# +# Thus you **MUST** check if any imports you add in this file are only +# referenced in this file (or only in any other core plugin), and if so... +# +# YOU MUST ENSURE THAT PERTINENT ADJUSTMENTS ARE MADE IN `setup.py` +# SO AS TO ENSURE THE FILES ARE ACTUALLY PRESENT IN AN END-USER +# INSTALLATION ON WINDOWS. +# +# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# import base64 import gzip import json diff --git a/plugins/inara.py b/plugins/inara.py index 08b3b86d..2446693a 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -1,5 +1,27 @@ """Inara Sync.""" +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# +# This is an EDMC 'core' plugin. +# +# All EDMC plugins are *dynamically* loaded at run-time. +# +# We build for Windows using `py2exe`. +# +# `py2exe` can't possibly know about anything in the dynamically loaded +# core plugins. +# +# Thus you **MUST** check if any imports you add in this file are only +# referenced in this file (or only in any other core plugin), and if so... +# +# YOU MUST ENSURE THAT PERTINENT ADJUSTMENTS ARE MADE IN `setup.py` +# SO AS TO ENSURE THE FILES ARE ACTUALLY PRESENT IN AN END-USER +# INSTALLATION ON WINDOWS. +# +# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# import json import threading import time From b108456ae3e6efe3d16ea71aa25783301502190c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 1 Feb 2022 15:26:10 +0000 Subject: [PATCH 3/3] Contributing: Add new section about plugin-only imports --- Contributing.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/Contributing.md b/Contributing.md index 100b43ff..4d7fc9fc 100644 --- a/Contributing.md +++ b/Contributing.md @@ -246,6 +246,41 @@ Otherwise, see the [pytest documentation](https://docs.pytest.org/en/stable/cont --- +## Imports used only in core plugins + +Because the 'core' plugins, as with any EDMarketConnector plugin, are only ever +loaded dynamically, not through an explicit `import` statement, there is no +way for `py2exe` to know about them when building the contents of the +`dist.win32` directory. See [docs/Releasing.md](docs/Releasing.md) for more +information about this build process. + +Thus, you **MUST** check if any imports you add in `plugins/*.py` files are only +referenced in that file (or also only in any other core plugin), and if so +**YOU MUST ENSURE THAT PERTINENT ADJUSTMENTS ARE MADE IN `setup.py` +IN ORDER TO ENSURE THE FILES ARE ACTUALLY PRESENT IN AN END-USER +INSTALLATION ON WINDOWS.** + +An exmaple is that as of 2022-02-01 it was noticed that `plugins/eddn.py` now +uses `util/text.py`, and is the only code to do so. `py2exe` does not detect +this and thus the resulting `dist.win32/library.zip` does not contain the +`util/` directory, let alone the `util/text.py` file. The fix was to update +the appropriate `packages` definition to: + +```python + 'packages': [ + 'sqlite3', # Included for plugins + 'util', # 2022-02-01 only imported in plugins/eddn.py + ], +``` + +Note that in this case it's in `packages` because we want the whole directory +adding. For a single file an extra item in `includes` would suffice. + +Such additions to `setup.py` should not cause any issues if subsequent project +changes cause `py2exe` to automatically pick up the same file(s). + +--- + ## Debugging network sends Rather than risk sending bad data to a remote service, even if only through