chore: fix CI MaaDeps download (#3884)

.github/workflows/dev-build-linux.yml

@@ -34,13 +34,16 @@ jobs:
         uses: Chocobo1/setup-ccache-action@v1
         with:
           remove_stale_cache: false
-
+      - name: Bootstrap MaaDeps
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 maadeps-download.py
       - name: Build MAA
         env:
           CC: ccache gcc-12
           CXX: ccache g++-12
         run: |
-          python3 maadeps-download.py
           mkdir -p build
           cmake -B build \
             -DINSTALL_THIRD_LIBS=ON \

.github/workflows/dev-build-mac.yml

@@ -41,6 +41,10 @@ jobs:
         run: |
           brew update --preinstall
           brew install ninja pkg-config range-v3
+      - name: Bootstrap MaaDeps
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
           python3 maadeps-download.py
       - name: Configure MaaCore
         run: |

.github/workflows/dev-build-win.yml

@@ -73,9 +73,13 @@ jobs:
             .nuke/temp
             ~/.nuget/packages
           key: ${{ runner.os }}-${{ matrix.msbuild_target }}-${{ hashFiles('**/global.json', '**/*.csproj') }}
+      - name: Bootstrap MaaDeps
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 maadeps-download.py ${{ matrix.lowercase_target }}-windows
       - name: Run './build.cmd DevBuild'
         run:  |
-          python3 maadeps-download.py ${{ matrix.lowercase_target }}-windows
           ./build.cmd DevBuild
         env:
           Reason: ${{ github.event.inputs.Reason }}

.github/workflows/release-maa-linux.yml

@@ -21,13 +21,16 @@ jobs:
         uses: Chocobo1/setup-ccache-action@v1
         with:
           remove_stale_cache: false
-
+      - name: Bootstrap MaaDeps
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 maadeps-download.py
       - name: Build MAA
         env:
           CC: ccache gcc-12
           CXX: ccache g++-12
         run: |
-          python3 maadeps-download.py
           mkdir -p build
           cmake -B build \
             -DINSTALL_THIRD_LIBS=ON \

.github/workflows/release-maa-mac.yml

@@ -31,8 +31,12 @@ jobs:
         run: |
             brew update --preinstall
             brew install create-dmg ninja pkg-config range-v3
-            [[ ${{ matrix.arch }} = "arm64" ]] && triplet="arm64-osx" || triplet="x64-osx"
-            python3 maadeps-download.py ${triplet}
+      - name: Bootstrap MaaDeps
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          [[ ${{ matrix.arch }} = "arm64" ]] && triplet="arm64-osx" || triplet="x64-osx"
+          python3 maadeps-download.py ${triplet}
       - name: Configure MaaCore
         run: |
           cmake -B build -GNinja -DCMAKE_BUILD_TYPE=Release -DBUILD_XCFRAMEWORK=ON -DCMAKE_OSX_ARCHITECTURES=${{ matrix.arch }}

.github/workflows/release-maa-win.yml

@@ -44,9 +44,13 @@ jobs:
             .nuke/temp
             ~/.nuget/packages
           key: ${{ runner.os }}-${{ matrix.msbuild_target }}-${{ hashFiles('**/global.json', '**/*.csproj') }}
-      - name: Run './build.cmd ReleaseMaa'
+      - name: Bootstrap MaaDeps
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           python3 maadeps-download.py ${{ matrix.lowercase_target }}-windows
+      - name: Run './build.cmd ReleaseMaa'
+        run: |
           ./build.cmd ReleaseMaa
         env:
           PUBLISH_GH_PAT: ${{ secrets.PUBLISH_GH_PAT }}

.github/workflows/release-nightly-ota.yml

@@ -26,24 +26,11 @@ on:
         default: 30
 
 jobs:
-  build-win-nightly:
+  set-tag:
     runs-on: windows-latest
     outputs:
       tag: ${{ steps.set_tag.outputs.tag }}
     steps:
-      - uses: actions/checkout@v3
-        with:
-          # repository: 'MaaAssistantArknights/MaaAssistantArknights'
-          submodules: recursive
-          #ref: ${{ inputs.ref }}
-          fetch-depth: 0
-      - name: Checkout ref
-        run: |
-          if ("${{ inputs.ref }}" -ne "") {
-            git checkout --progress --recurse-submodules ${{ inputs.ref || 'dev' }}
-          }
-      - run: |
-          npm install --global --progress semver
       - name: Set tag
         id: set_tag
         run: |
@@ -63,6 +50,34 @@ jobs:
             exit 0
           }
           exit 1
+  build-win-nightly:
+    needs: set-tag
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        include:
+          - msbuild_target: x64
+            lowercase_target: x64
+          - msbuild_target: ARM64
+            lowercase_target: arm64
+      fail-fast: false
+    env:
+      MAABUILDER_TARGET_PLATFORM: ${{ matrix.msbuild_target }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # repository: 'MaaAssistantArknights/MaaAssistantArknights'
+          submodules: recursive
+          #ref: ${{ inputs.ref }}
+          fetch-depth: 0
+      - name: Checkout ref
+        run: |
+          if ("${{ inputs.ref }}" -ne "") {
+            git checkout --progress --recurse-submodules ${{ inputs.ref || 'dev' }}
+          }
+      - run: |
+          npm install --global --progress semver
+
       - name: Cache .nuke/temp, ~/.nuget/packages
         uses: actions/cache@v3
         with:
@@ -70,27 +85,34 @@ jobs:
             .nuke/temp
             ~/.nuget/packages
           key: ${{ runner.os }}-${{ hashFiles('**/global.json', '**/*.csproj') }}
+      - name: Bootstrap MaaDeps
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 maadeps-download.py ${{ matrix.lowercase_target }}-windows
       - name: Run './build.cmd DevBuild'
         run: |
           $env:GITHUB_WORKFLOW = 'dev-build-win' # pretend this is a dev-build-win workflow
-          $env:MAA_BUILDER_MAA_VERSION = "${{steps.set_tag.outputs.tag}}"
+          $env:MAA_BUILDER_MAA_VERSION = "${{ needs.set-tag.outputs.tag }}"
           echo "tag: " $env:MAA_BUILDER_MAA_VERSION
 
-          python3 maadeps-download.py x64-windows
           ./build.cmd DevBuild
         env:
           Reason: 'Build nightly version'
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/upload-artifact@v3
         with:
-          name: MAA-win-x64
+          name: MAA-win-${{ matrix.lowercase_target }}
           path: artifacts
 
   make-ota:
-    needs: build-win-nightly
+    needs: [set-tag, build-win-nightly]
+    strategy:
+      matrix:
+        target: [x64, arm64]
     runs-on: ubuntu-latest
     steps:
-      - run: echo ${{ needs.build-win-nightyl.outputs.tag }}
+      - run: echo ${{ needs.set-tag.outputs.tag }}
       - name: "Fetch MaaRelease"
         uses: actions/checkout@v3
         with:
@@ -103,7 +125,7 @@ jobs:
           path: MaaAssistantArknights
       - uses: actions/download-artifact@v3
         with:
-          name: MAA-win-x64
+          name: MAA-win-${{ matrix.target }}
           path: ${{ format('{0}/{1}', 'build-ota', needs.build-win-nightly.outputs.tag) }}
       - name: "Fetch release info"
         env:
@@ -133,8 +155,8 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           cd build-ota
-          $GITHUB_WORKSPACE/MaaAssistantArknights/tools/OTAPacker/build.sh 'MaaAssistantArknights/MaaAssistantArknights' ./config "${{ github.repository_owner }}/MaaRelease"
-          mv -v ${{ needs.build-win-nightly.outputs.tag }}/*.zip ./MAA-${{ env.release_tag }}-win-x64.zip
+          $GITHUB_WORKSPACE/MaaAssistantArknights/tools/OTAPacker/build.sh 'MaaAssistantArknights/MaaAssistantArknights' ./config ${{ matrix.target }} "${{ github.repository_owner }}/MaaRelease"
+          mv -v ${{ needs.build-win-nightly.outputs.tag }}/*.zip ./MAA-${{ env.release_tag }}-win-${{ matrix.target }}.zip
       - name: "Commit and setup tag"
         run: |
           cd MaaRelease

.github/workflows/release-ota.yml

@@ -18,6 +18,8 @@ on:
 jobs:
   make-ota:
     runs-on: "ubuntu-latest"
+    matrix: 
+      target: [x64, arm64]
     steps:
       - name: "Fetch MaaRelease"
         uses: actions/checkout@v3
@@ -53,7 +55,7 @@ jobs:
           mkdir -pv build-ota/${{ env.release_tag }}
           cd build-ota/${{ env.release_tag }}
           mkdir -pv content
-          gh release download ${{ env.release_tag }} --repo 'MaaAssistantArknights/MaaAssistantArknights' --pattern "MAA-${{ env.release_tag }}-win-x64.zip" --clobber
+          gh release download ${{ env.release_tag }} --repo 'MaaAssistantArknights/MaaAssistantArknights' --pattern "MAA-${{ env.release_tag }}-win-${{ matrix.target }}.zip" --clobber
           unzip -q -O GB2312 -o "*.zip" -d 'content'
           mv *.zip ..
         env:
@@ -63,7 +65,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           cd build-ota
-          $GITHUB_WORKSPACE/MaaAssistantArknights/tools/OTAPacker/build.sh 'MaaAssistantArknights/MaaAssistantArknights' ./config 'MaaAssistantArknights/MaaRelease'
+          $GITHUB_WORKSPACE/MaaAssistantArknights/tools/OTAPacker/build.sh 'MaaAssistantArknights/MaaAssistantArknights' ./config ${{ matrix.target }} 'MaaAssistantArknights/MaaRelease'
       - name: "Commit and setup tag"
         run: |
           cd MaaRelease

maadeps-build.py

@@ -8,6 +8,8 @@ basedir = Path(__file__).parent
 def main():
     os.chdir(basedir)
     subprocess.check_call(["git", "submodule", "update", "--init", "--recommend-shallow", "--remote", "MaaDeps"])
-    subprocess.check_call(["git", "submodule", "update", "--init", "--recommend-shallow", "MaaDeps/vcpkg"])
     ret = subprocess.call([sys.executable, "build.py", *sys.argv[1:]], cwd=Path(basedir, "MaaDeps"))
     sys.exit(ret)
+
+if __name__ == '__main__':
+    main()

maadeps-download.py

@@ -70,25 +70,27 @@ def sanitize_filename(filename: str):
     return filename
 
 
-def retry_load(url: str):
+def retry_urlopen(*args, **kwargs):
     import time
     import http.client
     for _ in range(5):
-        resp: http.client.HTTPResponse = urllib.request.urlopen(url)
-        if resp.status == 403 and resp.headers.get("x-ratelimit-remaining") == "0":
-            # rate limit
-            resp.close()
-            t0 = time.time()
-            reset_time = t0 + 10
-            try:
-                reset_time = int(resp.headers.get("x-ratelimit-reset", 0))
-            except ValueError:
-                pass
-            reset_time = max(reset_time, t0 + 10)
-            print(f"rate limit exceeded, retrying after {reset_time - t0:.1f} seconds")
-            time.sleep(reset_time - t0)
-            continue
-        return resp.read()
+        try:
+            resp: http.client.HTTPResponse = urllib.request.urlopen(*args, **kwargs)
+            return resp
+        except urllib.error.HTTPError as e:
+            if e.status == 403 and e.headers.get("x-ratelimit-remaining") == "0":
+                # rate limit
+                t0 = time.time()
+                reset_time = t0 + 10
+                try:
+                    reset_time = int(e.headers.get("x-ratelimit-reset", 0))
+                except ValueError:
+                    pass
+                reset_time = max(reset_time, t0 + 10)
+                print(f"rate limit exceeded, retrying after {reset_time - t0:.1f} seconds")
+                time.sleep(reset_time - t0)
+                continue
+            raise
 
 
 def main():
@@ -99,8 +101,11 @@ def main():
     if len(sys.argv) == 1:
         print(f"to specify another triplet, run `{sys.argv[0]} <target triplet>`")
         print(f"e.g. `{sys.argv[0]} arm64-windows`")
-    
-    resp = retry_load("https://api.github.com/repos/MaaAssistantArknights/MaaDeps/releases")
+    req = urllib.request.Request("https://api.github.com/repos/MaaAssistantArknights/MaaDeps/releases")
+    token = os.environ.get("GH_TOKEN", os.environ.get("GITHUB_TOKEN", None))
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    resp = retry_urlopen(req).read()
     releases = json.loads(resp)
     def split_asset_name(name: str):
         *remainder, component_suffix = name.rsplit('-', 1)

tools/OTAPacker/build.sh

@@ -2,7 +2,8 @@
 
 source_repo=$1
 releases_txt="$2"
-source_repo_fallback=$3
+arch=$3
+source_repo_fallback=$4
 
 working_dir="$(pwd)"
 echo "working_dir: $working_dir"
@@ -16,8 +17,8 @@ while read tag; do
         mkdir -pv "$tag"
         cd "$tag"
         echo "Downloading $tag"
-        gh release download "$tag" --repo $source_repo --pattern "MAA-$tag-win-x64.zip" --pattern "MaaBundle-$tag.zip" --clobber \
-            || gh release download "$tag" --repo $source_repo_fallback --pattern "MAA-$tag-win-x64.zip" --pattern "MaaBundle-$tag.zip" --clobber
+        gh release download "$tag" --repo $source_repo --pattern "MAA-$tag-win-$arch.zip" --clobber \
+            || gh release download "$tag" --repo $source_repo_fallback --pattern "MAA-$tag-win-$arch.zip" --clobber
         mkdir -pv 'content'
         unzip -q -O GB2312 -o "*.zip" -d 'content'
         rm -fv *.zip
@@ -36,7 +37,7 @@ while read tag; do
         "$script_dir"/makeota.sh "$tag"/content "$latest_tag"/content "$tag"/pkg
         echo "Creating zip archive"
         cd "$tag"/pkg
-        zip -q -9 -r "$working_dir"/"MAAComponent-OTA-${tag}_${latest_tag}-win-x64.zip" .
+        zip -q -9 -r "$working_dir"/"MAAComponent-OTA-${tag}_${latest_tag}-win-$arch.zip" .
         cd $working_dir
     fi
 done < "$releases_txt"