From a67609696c7699ee529055d9bf11c16e22a69d23 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 11:36:54 +0000 Subject: [PATCH 01/21] windows-build: Properly name the build job --- .github/workflows/windows-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 93091a7c..a64c5306 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: jobs: - test: + windows_build: name: Build EDMC runs-on: windows-2019 From c8e6285a136347e87d0b89bc71688b342be6b73f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 12:05:53 +0000 Subject: [PATCH 02/21] windows-build: Create a custom release .zip --- .github/workflows/windows-build.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index a64c5306..232a51e7 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -7,7 +7,20 @@ on: workflow_dispatch: jobs: + variables: + outputs: + sem_ver: ${{ steps.var.outputs.sem_ver }} + runs-on: "ubuntu-latest" + steps: + - name: Setting global variables + uses: actions/github-script@v6 + id: var + with: + script: | + core.setOutput('sem_ver', '${{ github.ref_name }}'.replaceAll('Release\/', '')) + windows_build: + needs: [variables] name: Build EDMC runs-on: windows-2019 @@ -40,11 +53,20 @@ jobs: run: | python Build-exe-and-msi.py + - name: Make zip archive + uses: thedoctor0/zip-release@main + with: + type: 'zip' + filename: 'EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' + exclusions: 'EDMarketConnector-release-*.zip .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data/* img/* pyproject.toml scripts/* tests/* wix/*' + - name: Upload build files uses: actions/upload-artifact@v3 with: name: Built files - path: EDMarketConnector_win*.msi + path: | + EDMarketConnector_win*.msi + EDMarketConnector-release-*.zip release: name: Release new version @@ -60,7 +82,7 @@ jobs: path: ./ - name: Hash files - run: sha256sum EDMarketConnector_win*.msi > ./hashes.sum + run: sha256sum EDMarketConnector_win*.msi EDMarketConnector-release*.zip > ./hashes.sum - name: Create Draft Release uses: "softprops/action-gh-release@v1" @@ -71,4 +93,5 @@ jobs: discussion_category_name: "Announcement" files: | ./EDMarketConnector_win*.msi + ./EDMarketConnector-release-*.zip ./hashes.sum From c48cfeca5dd7b2368850908600233d3dd4b4ff99 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 12:12:30 +0000 Subject: [PATCH 03/21] windows-build: Fix 'needs' reference to windows_build job --- .github/workflows/windows-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 232a51e7..127fa243 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -71,7 +71,7 @@ jobs: release: name: Release new version runs-on: ubuntu-latest - needs: test + needs: windows_build if: "${{ github.event_name != 'workflow_dispatch' }}" steps: From 35b5a90409b03117d411ac1a06796810d8da6056 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 12:27:17 +0000 Subject: [PATCH 04/21] windows-build: Custom zip: Include top-level directory, and exclude more --- .github/workflows/windows-build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 127fa243..6ed99fd1 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -57,8 +57,10 @@ jobs: uses: thedoctor0/zip-release@main with: type: 'zip' + directory: '..' + path: 'EDMarketConnector' filename: 'EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' - exclusions: 'EDMarketConnector-release-*.zip .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data/* img/* pyproject.toml scripts/* tests/* wix/*' + exclusions: 'dist.win32/* dist.win32 __pycache__/* __pycache__ .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data/* coriolis-data/.??* coriolis-data img/* img out/* out out.zip pyproject.toml scripts/* scripts tests/* tests wix/* wix WinSparkle.*' - name: Upload build files uses: actions/upload-artifact@v3 @@ -66,7 +68,7 @@ jobs: name: Built files path: | EDMarketConnector_win*.msi - EDMarketConnector-release-*.zip + ../EDMarketConnector-release-*.zip release: name: Release new version From 2531e6de23cb8e58561ae8504e4ed8d2a0944bd8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 12:38:08 +0000 Subject: [PATCH 05/21] windows-build: Can't use relative path for upload-artifact So, let's *hope* that zip-release still puts the .zip it creates in the main directory, despite being told to create it from the parent. --- .github/workflows/windows-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 6ed99fd1..480e54ca 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -60,7 +60,7 @@ jobs: directory: '..' path: 'EDMarketConnector' filename: 'EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' - exclusions: 'dist.win32/* dist.win32 __pycache__/* __pycache__ .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data/* coriolis-data/.??* coriolis-data img/* img out/* out out.zip pyproject.toml scripts/* scripts tests/* tests wix/* wix WinSparkle.*' + exclusions: 'EDMarketConnector-release-*.zip dist.win32/* dist.win32 __pycache__/* __pycache__ .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data/* coriolis-data/.??* coriolis-data img/* img out/* out out.zip pyproject.toml scripts/* scripts tests/* tests wix/* wix WinSparkle.*' - name: Upload build files uses: actions/upload-artifact@v3 @@ -68,7 +68,7 @@ jobs: name: Built files path: | EDMarketConnector_win*.msi - ../EDMarketConnector-release-*.zip + EDMarketConnector-release-*.zip release: name: Release new version From 531c839d136c816320e6640a2b5fad6973b3f96e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 12:46:28 +0000 Subject: [PATCH 06/21] windows-build: Specify custom zip to be in the 'root' directory zip-release does indeed create this in `directory`, so we need to specify it should be *in* the 'root' directory that we're including *in* the zip file. --- .github/workflows/windows-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 480e54ca..958e5bf8 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -59,7 +59,7 @@ jobs: type: 'zip' directory: '..' path: 'EDMarketConnector' - filename: 'EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' + filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' exclusions: 'EDMarketConnector-release-*.zip dist.win32/* dist.win32 __pycache__/* __pycache__ .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data/* coriolis-data/.??* coriolis-data img/* img out/* out out.zip pyproject.toml scripts/* scripts tests/* tests wix/* wix WinSparkle.*' - name: Upload build files From 0c0c14166deeeca411e2af51745036cbfab80635 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 12:58:05 +0000 Subject: [PATCH 07/21] windows-build: custom zip exclusions need 'root' prefix Also, comment out exactly what we're doing in the zip-release configuration, and why. --- .github/workflows/windows-build.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 958e5bf8..148e8048 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -57,10 +57,18 @@ jobs: uses: thedoctor0/zip-release@main with: type: 'zip' + # We want an in-zip prefix of `EDMarketConnector/` for all files, so + # we specify that: + # 1. We work from the parent directory directory: '..' + # 2. The path we're using is the 'root' directory path: 'EDMarketConnector' + # 3. The .zip file has to be in the 'root' so that upload-artifact + # will process it. Can't use relative paths. filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' - exclusions: 'EDMarketConnector-release-*.zip dist.win32/* dist.win32 __pycache__/* __pycache__ .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data/* coriolis-data/.??* coriolis-data img/* img out/* out out.zip pyproject.toml scripts/* scripts tests/* tests wix/* wix WinSparkle.*' + # 4. And all the exclusions need to be prefixed with the 'root' + # directory. + exclusions: 'EDMarketConnector/EDMarketConnector-release-*.zip EDMarketConnector/dist.win32/* EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/* EDMarketConnector/__pycache__ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/* EDMarketConnector/coriolis-data/.??* EDMarketConnector/coriolis-data EDMarketConnector/img/* EDMarketConnector/img EDMarketConnector/out/* EDMarketConnector/out EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/* EDMarketConnector/scripts EDMarketConnector/tests/* EDMarketConnector/tests EDMarketConnector/wix/* EDMarketConnector/wix EDMarketConnector/WinSparkle.*' - name: Upload build files uses: actions/upload-artifact@v3 From 804b450c82b4b6caa359f9d9c4594fecb25f0c5d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 13:19:04 +0000 Subject: [PATCH 08/21] windows-build: Custom zip: Reduce exclusions It *seems* we don't need to specify both a directory and its contents. Also, hopefully this will fix the: Creating zip archive... Command Line Error: Unknown switch: - which the last run got. That was where the 'root' directory prefix was added to all exclusions. Maybe the commandline got too long ? --- .github/workflows/windows-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 148e8048..f2eda9fd 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -68,7 +68,7 @@ jobs: filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' # 4. And all the exclusions need to be prefixed with the 'root' # directory. - exclusions: 'EDMarketConnector/EDMarketConnector-release-*.zip EDMarketConnector/dist.win32/* EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/* EDMarketConnector/__pycache__ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/* EDMarketConnector/coriolis-data/.??* EDMarketConnector/coriolis-data EDMarketConnector/img/* EDMarketConnector/img EDMarketConnector/out/* EDMarketConnector/out EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/* EDMarketConnector/scripts EDMarketConnector/tests/* EDMarketConnector/tests EDMarketConnector/wix/* EDMarketConnector/wix EDMarketConnector/WinSparkle.*' + exclusions: 'EDMarketConnector/EDMarketConnector-release-*.zip EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*' - name: Upload build files uses: actions/upload-artifact@v3 From 64b1a4afd6c2ca258364dfd946ae0ec643689c0b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 11 Jan 2023 18:10:18 +0000 Subject: [PATCH 09/21] windows-build/custom zip: Add comment about the "too long args" issue --- .github/workflows/windows-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index f2eda9fd..70cca876 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -68,6 +68,8 @@ jobs: filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' # 4. And all the exclusions need to be prefixed with the 'root' # directory. + # NB: If this gets too long it can cause zip 'Command Line Error', + # presumably due to a Windows CL length limit. exclusions: 'EDMarketConnector/EDMarketConnector-release-*.zip EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*' - name: Upload build files From 4edebf981f3242361e8f87e9b1ccfff1b60594df Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 11:39:35 +0000 Subject: [PATCH 10/21] windows-build/archives: Create .tar.gz as well * Move the definition of archive excludes to a global variable. * Add another step to create a tar(.gz) archive as well. * Add .tar.gz to the `Built files` handling. --- .github/workflows/windows-build.yml | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 70cca876..cbee180f 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -10,6 +10,7 @@ jobs: variables: outputs: sem_ver: ${{ steps.var.outputs.sem_ver }} + archive_exclusions: ${{ steps.var.outputs.archive_exclusions }} runs-on: "ubuntu-latest" steps: - name: Setting global variables @@ -18,6 +19,11 @@ jobs: with: script: | core.setOutput('sem_ver', '${{ github.ref_name }}'.replaceAll('Release\/', '')) + # If this value gets too long it can cause issues with Windows + # command-line length and zip/tar will fail + # And all the exclusions need to be prefixed with the 'root' + # directory. + core.setOutput('archive_exclusions', 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*') windows_build: needs: [variables] @@ -70,15 +76,25 @@ jobs: # directory. # NB: If this gets too long it can cause zip 'Command Line Error', # presumably due to a Windows CL length limit. - exclusions: 'EDMarketConnector/EDMarketConnector-release-*.zip EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*' + exclusions: ${{ needs.variables.outputs.archive_exclusions }} + + - name: Make tar archive + uses: thedoctor0/zip-release@main + with: + # See the 'zip' version above for commentary on these values + type: 'tar' + path: 'EDMarketConnector' + filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz' + exclusions: ${{ needs.variables.outputs.archive_exclusions }} - name: Upload build files uses: actions/upload-artifact@v3 with: name: Built files path: | - EDMarketConnector_win*.msi + EDMarketConnector_win_*.msi EDMarketConnector-release-*.zip + EDMarketConnector-release-*.tar.gz release: name: Release new version @@ -94,7 +110,7 @@ jobs: path: ./ - name: Hash files - run: sha256sum EDMarketConnector_win*.msi EDMarketConnector-release*.zip > ./hashes.sum + run: sha256sum EDMarketConnector_win_*.msi EDMarketConnector-release-*.{zip,tar.gz} > ./hashes.sum - name: Create Draft Release uses: "softprops/action-gh-release@v1" @@ -104,6 +120,7 @@ jobs: prerelease: true discussion_category_name: "Announcement" files: | - ./EDMarketConnector_win*.msi + ./EDMarketConnector_win_*.msi ./EDMarketConnector-release-*.zip + ./EDMarketConnector-release-*.tar.gz ./hashes.sum From 43d10ab29a7ae55b52f90aa484070531d319b748 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 11:53:17 +0000 Subject: [PATCH 11/21] windows-build/archives: Can't multiple setOutput, use env instead * As the archive exclusions is static, we can just set it in workflow env. --- .github/workflows/windows-build.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index cbee180f..e55a8d5f 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -1,3 +1,4 @@ +# vim: tabstop=2 shiftwidth=2 name: Build EDMC for Windows on: @@ -6,6 +7,13 @@ on: - "Release/*" workflow_dispatch: +env: + # If this value gets too long it can cause issues with Windows + # command-line length and zip/tar will fail + # And all the exclusions need to be prefixed with the 'root' + # directory. + ARCHIVE_EXCLUSIONS: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*' + jobs: variables: outputs: @@ -19,11 +27,6 @@ jobs: with: script: | core.setOutput('sem_ver', '${{ github.ref_name }}'.replaceAll('Release\/', '')) - # If this value gets too long it can cause issues with Windows - # command-line length and zip/tar will fail - # And all the exclusions need to be prefixed with the 'root' - # directory. - core.setOutput('archive_exclusions', 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*') windows_build: needs: [variables] @@ -76,7 +79,7 @@ jobs: # directory. # NB: If this gets too long it can cause zip 'Command Line Error', # presumably due to a Windows CL length limit. - exclusions: ${{ needs.variables.outputs.archive_exclusions }} + exclusions: ${{ env.ARCHIVE_EXCLUSIONS }} - name: Make tar archive uses: thedoctor0/zip-release@main @@ -85,7 +88,7 @@ jobs: type: 'tar' path: 'EDMarketConnector' filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz' - exclusions: ${{ needs.variables.outputs.archive_exclusions }} + exclusions: ${{ env.ARCHIVE_EXCLUSIONS }} - name: Upload build files uses: actions/upload-artifact@v3 From 709847990267fea34e1fff8cd8c75061495785df Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 12:03:51 +0000 Subject: [PATCH 12/21] windows-build/tar: Different specification of exclusions --- .github/workflows/windows-build.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index e55a8d5f..0fcada1f 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -7,13 +7,6 @@ on: - "Release/*" workflow_dispatch: -env: - # If this value gets too long it can cause issues with Windows - # command-line length and zip/tar will fail - # And all the exclusions need to be prefixed with the 'root' - # directory. - ARCHIVE_EXCLUSIONS: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*' - jobs: variables: outputs: @@ -79,7 +72,7 @@ jobs: # directory. # NB: If this gets too long it can cause zip 'Command Line Error', # presumably due to a Windows CL length limit. - exclusions: ${{ env.ARCHIVE_EXCLUSIONS }} + exclusions: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*' - name: Make tar archive uses: thedoctor0/zip-release@main @@ -88,7 +81,8 @@ jobs: type: 'tar' path: 'EDMarketConnector' filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz' - exclusions: ${{ env.ARCHIVE_EXCLUSIONS }} + # For 'tar' we can only specify filenames, not any directory location + exclusions: 'EDMarketConnector-release-*.* dist.win32 __pycache__ .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data img out out.zip pyproject.toml scripts tests wix WinSparkle.*' - name: Upload build files uses: actions/upload-artifact@v3 From 15ee894142658927924f4feed9bf0b6efaae99e9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 12:14:00 +0000 Subject: [PATCH 13/21] windows-build/archives: Re-order to very early, add `directory` to tar * I'd missed copying the `directory` line from zip to tar step. * There's no reason we can't perform these steps *before* the build, or even the python setup, so move them up there. That has the near side-effect of not needing to exclude so much stuff. --- .github/workflows/windows-build.yml | 59 +++++++++++++++-------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 0fcada1f..d5a40656 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -35,6 +35,36 @@ jobs: with: submodules: true + - name: Make tar archive + uses: thedoctor0/zip-release@main + with: + # See the 'zip' version for commentary on these values + type: 'tar' + directory: '..' + path: 'EDMarketConnector' + filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz' + # For 'tar' we can only specify filenames, not any directory location + exclusions: 'EDMarketConnector-release-*.* .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data img pyproject.toml scripts tests wix' + + - name: Make zip archive + uses: thedoctor0/zip-release@main + with: + type: 'zip' + # We want an in-zip prefix of `EDMarketConnector/` for all files, so + # we specify that: + # 1. We work from the parent directory + directory: '..' + # 2. The path we're using is the 'root' directory + path: 'EDMarketConnector' + # 3. The .zip file has to be in the 'root' so that upload-artifact + # will process it. Can't use relative paths. + filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' + # 4. And all the exclusions need to be prefixed with the 'root' + # directory. + # NB: If this gets too long it can cause zip 'Command Line Error', + # presumably due to a Windows CL length limit. + exclusions: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/' + - uses: actions/setup-python@v4 with: python-version-file: '.python-version' @@ -55,35 +85,6 @@ jobs: run: | python Build-exe-and-msi.py - - name: Make zip archive - uses: thedoctor0/zip-release@main - with: - type: 'zip' - # We want an in-zip prefix of `EDMarketConnector/` for all files, so - # we specify that: - # 1. We work from the parent directory - directory: '..' - # 2. The path we're using is the 'root' directory - path: 'EDMarketConnector' - # 3. The .zip file has to be in the 'root' so that upload-artifact - # will process it. Can't use relative paths. - filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.zip' - # 4. And all the exclusions need to be prefixed with the 'root' - # directory. - # NB: If this gets too long it can cause zip 'Command Line Error', - # presumably due to a Windows CL length limit. - exclusions: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/dist.win32 EDMarketConnector/__pycache__/ EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/out/ EDMarketConnector/out.zip EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/ EDMarketConnector/WinSparkle.*' - - - name: Make tar archive - uses: thedoctor0/zip-release@main - with: - # See the 'zip' version above for commentary on these values - type: 'tar' - path: 'EDMarketConnector' - filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz' - # For 'tar' we can only specify filenames, not any directory location - exclusions: 'EDMarketConnector-release-*.* dist.win32 __pycache__ .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data img out out.zip pyproject.toml scripts tests wix WinSparkle.*' - - name: Upload build files uses: actions/upload-artifact@v3 with: From 3e05f919494c910e279b5e269a271f35b5d99e3d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 12:18:09 +0000 Subject: [PATCH 14/21] windows-build/tar: Now we have `directory` we *do* need exclusion paths * Maybe I was wrong about not needing the prefix path on tar exclusions, because that was when I'd not copied `directory`. --- .github/workflows/windows-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index d5a40656..19d836af 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -44,7 +44,8 @@ jobs: path: 'EDMarketConnector' filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz' # For 'tar' we can only specify filenames, not any directory location - exclusions: 'EDMarketConnector-release-*.* .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data img pyproject.toml scripts tests wix' + # exclusions: 'EDMarketConnector-release-*.* .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data img pyproject.toml scripts tests wix' + exclusions: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/' - name: Make zip archive uses: thedoctor0/zip-release@main From 042c8d4ff243433ecbd0529c243a0570f1c7291a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 12:34:44 +0000 Subject: [PATCH 15/21] windows-build/tar: Move to running under ubuntu * On windows specifying only filenames to exclusions causes things like `tar: .editorconfig: Cannot stat: No such file or directory` But putting `EDMarketConnector/` prefix on such means they don't work. * Testing with 'git bash' GNU tar 1.34: `tar -c -v -z -f edmc-test.tar.gz --exclude=.editorconfig --exclude=.git\* EDMarketConnector` works, with empty output from: `tar tfvz edmc-test.tar.gz | grep editorconfig` --- .github/workflows/windows-build.yml | 36 ++++++++++++++++++----------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 19d836af..ce6a9175 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -21,16 +21,6 @@ jobs: script: | core.setOutput('sem_ver', '${{ github.ref_name }}'.replaceAll('Release\/', '')) - windows_build: - needs: [variables] - name: Build EDMC - runs-on: windows-2019 - - defaults: - run: - shell: powershell - - steps: - uses: actions/checkout@v3 with: submodules: true @@ -44,8 +34,29 @@ jobs: path: 'EDMarketConnector' filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz' # For 'tar' we can only specify filenames, not any directory location - # exclusions: 'EDMarketConnector-release-*.* .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data img pyproject.toml scripts tests wix' - exclusions: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/' + exclusions: 'EDMarketConnector-release-*.* .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data img pyproject.toml scripts tests wix' + # exclusions: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/' + + - name: Upload build files + uses: actions/upload-artifact@v3 + with: + name: Built files + path: | + EDMarketConnector-release-*.tar.gz + + windows_build: + needs: [variables] + name: Build EDMC + runs-on: windows-2019 + + defaults: + run: + shell: powershell + + steps: + - uses: actions/checkout@v3 + with: + submodules: true - name: Make zip archive uses: thedoctor0/zip-release@main @@ -93,7 +104,6 @@ jobs: path: | EDMarketConnector_win_*.msi EDMarketConnector-release-*.zip - EDMarketConnector-release-*.tar.gz release: name: Release new version From 048be66f54a46a5a661703612e5da12505fff2fe Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 12:54:19 +0000 Subject: [PATCH 16/21] windows-build/tar: Just run a tar command ourselves directly thedoctor0/zip-release has a bug where only the first word in `exclusions` has `--exclude=` pre-pended to it, the other words being treated as input filenames/globs/paths. So, let's do this manually. --- .github/workflows/windows-build.yml | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index ce6a9175..c074d170 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -26,16 +26,27 @@ jobs: submodules: true - name: Make tar archive - uses: thedoctor0/zip-release@main with: - # See the 'zip' version for commentary on these values - type: 'tar' - directory: '..' - path: 'EDMarketConnector' - filename: 'EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz' - # For 'tar' we can only specify filenames, not any directory location - exclusions: 'EDMarketConnector-release-*.* .editorconfig .flake8 .git* .mypy.ini .pre-commit-config.yaml Build-exe-and-msi.py *.manifest coriolis-data img pyproject.toml scripts tests wix' - # exclusions: 'EDMarketConnector/EDMarketConnector-release-*.* EDMarketConnector/.editorconfig EDMarketConnector/.flake8 EDMarketConnector/.git* EDMarketConnector/.mypy.ini EDMarketConnector/.pre-commit-config.yaml EDMarketConnector/Build-exe-and-msi.py EDMarketConnector/*.manifest EDMarketConnector/coriolis-data/ EDMarketConnector/img/ EDMarketConnector/pyproject.toml EDMarketConnector/scripts/ EDMarketConnector/tests/ EDMarketConnector/wix/' + script: | + # For 'tar' we can only specify filename/glob exclusions, not any + # directory location + tar -C .. -c -v -z \ + -f EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz \ + --exclude=EDMarketConnector-release-*.* \ + --exclude=.editorconfig \ + --exclude=.flake8 \ + --exclude=.git* \ + --exclude=.mypy.ini \ + --exclude=.pre-commit-config.yaml \ + --exclude=Build-exe-and-msi.py \ + --exclude=*.manifest \ + --exclude=coriolis-data \ + --exclude=img \ + --exclude=pyproject.toml \ + --exclude=scripts \ + --exclude=tests \ + --exclude=wix \ + EDMarketConnector - name: Upload build files uses: actions/upload-artifact@v3 From d52f161dc94c6fff8f080460394383594f1ec01c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 12:58:10 +0000 Subject: [PATCH 17/21] windows-build/tar: `run`, not `script`. --- .github/workflows/windows-build.yml | 41 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index c074d170..a7a8a5da 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -26,27 +26,26 @@ jobs: submodules: true - name: Make tar archive - with: - script: | - # For 'tar' we can only specify filename/glob exclusions, not any - # directory location - tar -C .. -c -v -z \ - -f EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz \ - --exclude=EDMarketConnector-release-*.* \ - --exclude=.editorconfig \ - --exclude=.flake8 \ - --exclude=.git* \ - --exclude=.mypy.ini \ - --exclude=.pre-commit-config.yaml \ - --exclude=Build-exe-and-msi.py \ - --exclude=*.manifest \ - --exclude=coriolis-data \ - --exclude=img \ - --exclude=pyproject.toml \ - --exclude=scripts \ - --exclude=tests \ - --exclude=wix \ - EDMarketConnector + run: | + # For 'tar' we can only specify filename/glob exclusions, not any + # directory location + tar -C .. -c -v -z \ + -f EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz \ + --exclude=EDMarketConnector-release-*.* \ + --exclude=.editorconfig \ + --exclude=.flake8 \ + --exclude=.git* \ + --exclude=.mypy.ini \ + --exclude=.pre-commit-config.yaml \ + --exclude=Build-exe-and-msi.py \ + --exclude=*.manifest \ + --exclude=coriolis-data \ + --exclude=img \ + --exclude=pyproject.toml \ + --exclude=scripts \ + --exclude=tests \ + --exclude=wix \ + EDMarketConnector - name: Upload build files uses: actions/upload-artifact@v3 From a00c6f789a381572b6f53cb537774fcba12f7bd1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 13:03:06 +0000 Subject: [PATCH 18/21] windows-build/tar: Adjust argument ordering `-C ..` applies to anything that happens after it, so needs to be after the `-f `. --- .github/workflows/windows-build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index a7a8a5da..20b48c30 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -29,8 +29,9 @@ jobs: run: | # For 'tar' we can only specify filename/glob exclusions, not any # directory location - tar -C .. -c -v -z \ - -f EDMarketConnector/EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz \ + tar -c -v -z \ + -f EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz \ + -C .. \ --exclude=EDMarketConnector-release-*.* \ --exclude=.editorconfig \ --exclude=.flake8 \ From bcfcc8c71c78f24fdbe28c6caaf537ebe3dc0246 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 13:06:53 +0000 Subject: [PATCH 19/21] windows-build/tar: tar needs to be in job *after* `variables` At least we had `-f EDMarketConnector-release-.tar.gz ` which implies the `needs.variables.outputs.sem_ver` variable wasn't yet set. --- .github/workflows/windows-build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 20b48c30..ef1108ad 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -21,6 +21,9 @@ jobs: script: | core.setOutput('sem_ver', '${{ github.ref_name }}'.replaceAll('Release\/', '')) + linux_build: + runs-on: "ubuntu-latest" + steps: - uses: actions/checkout@v3 with: submodules: true From 2678d7d63f70b3e28cb86f7dbcdeb9a3e114aa08 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 13:10:43 +0000 Subject: [PATCH 20/21] windows-build/tar: New job needs to depend on `variables` --- .github/workflows/windows-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index ef1108ad..0cf0695a 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -22,6 +22,8 @@ jobs: core.setOutput('sem_ver', '${{ github.ref_name }}'.replaceAll('Release\/', '')) linux_build: + needs: [variables] + name: Linux environment build steps runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v3 From 48a80411b4c44f5b6365929f853a6035673bec3e Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 12 Jan 2023 13:47:41 +0000 Subject: [PATCH 21/21] windows-build/tar: Make archive at .. and then mv into 'root' * Because the archive is a new file in the root it causes: `tar: EDMarketConnector: file changed as we read it` Verified on Linux/Debian, and also verified that this fix works. --- .github/workflows/windows-build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 0cf0695a..3f306884 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -35,7 +35,7 @@ jobs: # For 'tar' we can only specify filename/glob exclusions, not any # directory location tar -c -v -z \ - -f EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz \ + -f ../EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz \ -C .. \ --exclude=EDMarketConnector-release-*.* \ --exclude=.editorconfig \ @@ -52,6 +52,7 @@ jobs: --exclude=tests \ --exclude=wix \ EDMarketConnector + mv ../EDMarketConnector-release-${{ needs.variables.outputs.sem_ver }}.tar.gz . - name: Upload build files uses: actions/upload-artifact@v3 @@ -124,7 +125,7 @@ jobs: release: name: Release new version runs-on: ubuntu-latest - needs: windows_build + needs: [ windows_build, linux_build ] if: "${{ github.event_name != 'workflow_dispatch' }}" steps: