Adding ForgeAPI version pinning and dependency download. (#1308)

Co-authored-by: christopher blodgett <christopher.blodgett@gmail.com>
Co-authored-by: Geoff Bourne <itzgeoff@gmail.com>

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
           cache-from: type=gha
 
       - name: Run Setup Only Tests
-        run: sh tests/setuponlytests/test.sh
+        run: bash tests/setuponlytests/test.sh
 
       # - name: Run Full Minecraft Service Tests
       #   run: |

README.md

@@ -756,6 +756,7 @@ Parameters to use the ForgeAPI:
 * `MODS_FORGEAPI_FILE` - Required or use MODS_FORGEAPI_PROJECTIDS (Overrides MODS_FORGEAPI_PROJECTIDS)
 * `MODS_FORGEAPI_PROJECTIDS` - Required or use MODS_FORGEAPI_FILE
 * `MODS_FORGEAPI_RELEASES` - Default is release, Options: [Release|Beta|Alpha]
+* `MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES` - Default is False, attempts to download required mods (releaseType Release) defined in Forge.
 * `REMOVE_OLD_FORGEAPI_MODS` - Default is False
 * `REMOVE_OLD_DATAPACKS_DEPTH` - Default is 1
 * `REMOVE_OLD_DATAPACKS_INCLUDE` - Default is *.jar
@@ -768,7 +769,13 @@ Example of expected forge api project ids, releases, and key:
   MODS_FORGEAPI_KEY: $WRX...
 ```
 
-Example of expected ForgeAPI file format: **Note**: name is currently unused, but can be used to document each entry.
+Example of expected ForgeAPI file format.
+
+**Field Description**: 
+* Name is currently unused, but can be used to document each entry.
+* Project id is the id found on the CurseForge website for a particular mod
+* Release Type corresponds to forge's R, B, A icon for each file. Default Release, options are (release|beta|alpha).
+* FileName is used for version pinning if latest file will not work for you.
 
 ```json
 [
@@ -781,6 +788,12 @@ Example of expected ForgeAPI file format: **Note**: name is currently unused, bu
       "name": "fabric voice mod",
       "projectId": "416089",
       "releaseType": "beta"
+  },
+  {
+      "name": "Biomes o plenty",
+      "projectId": "220318",
+      "fileName": "BiomesOPlenty-1.18.1-15.0.0.100-universal.jar",
+      "releaseType": "release"
   }
 ]
 ```

scripts/start-setupForgeApiMods

@@ -7,6 +7,7 @@ set -e -o pipefail
 : "${MODS_FORGEAPI_PROJECTIDS:=}"
 : "${MODS_FORGEAPI_FILE:=}"
 : "${MODS_FORGEAPI_RELEASES:=RELEASE}"
+: "${MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES:=false}"
 : "${REMOVE_OLD_MODS_DEPTH:=1} "
 : "${REMOVE_OLD_MODS_INCLUDE:=*.jar}"
 
@@ -36,11 +37,11 @@ ensureModKey(){
 # NOTE: downcasing release type for comparing types.
 updateReleaseNumber(){
   releaseType=$1
-  if [ "release" = "${releaseType,,}" ]; then
+  if [ "release" = "${releaseType,,}" ] || [ 1 = "${releaseType,,}" ]; then
     RELEASE_NUMBER_FILTER=1
-  elif [ "beta" = "${releaseType,,}" ]; then
+  elif [ "beta" = "${releaseType,,}" ] || [ 2 = "${releaseType,,}" ]; then
     RELEASE_NUMBER_FILTER=2
-  elif [ "alpha" = "${releaseType,,}" ]; then
+  elif [ "alpha" = "${releaseType,,}" ] || [ 3 = "${releaseType,,}" ]; then
     RELEASE_NUMBER_FILTER=3
   fi
 }
@@ -52,12 +53,12 @@ retrieveVersionTypeNumber(){
     -H 'Accept: application/json' -H 'x-api-key: '${MODS_FORGEAPI_KEY}'')
 
   if [ ! "$minecraft_types" ]; then
-    log "ERROR: unable to retrieve version types for ${VERSION_NAME} from ForgeAPI"
+    log "ERROR: unable to retrieve version types for ${VERSION_NAME} from ForgeAPI. Check Forge API key or supplied Minecraft version"
     exit 2
   fi
 
   TYPE_ID=$(jq -n "$minecraft_types" | jq --arg VERSION_NAME "$VERSION_NAME" -jc '
-      .data[] | select(.name==$VERSION_NAME) | .id')
+      .data[]? | select(.name==$VERSION_NAME) | .id')
 
   if [ ! "$TYPE_ID" ]; then
     log "ERROR: unable to retrieve version types for ${VERSION_NAME} from ForgeAPI"
@@ -68,6 +69,7 @@ retrieveVersionTypeNumber(){
 modFileByProjectID(){
   project_id=$(echo $1 | tr -d '"')
   project_id_release_type=$2
+  project_id_file_name=$3
   unset PROJECT_FILE
 
   # if Type id isn't defined use minecraft version to go get it.
@@ -100,9 +102,13 @@ modFileByProjectID(){
     fi
     # grabs the highest ID of the releaseTypes selected.
     # Default is 1 for Release, Beta is 2, and Alpha is 3. Using less than we can validate highest release.
-    current_project_file=$(jq -n "$project_files" | jq --arg RELEASE_FILTER "$RELEASE_NUMBER_FILTER" -jc '
-      .data | sort_by(.id) | reverse | map(select(.releaseType<=($RELEASE_FILTER|tonumber))) | .[0]')
-
+    if [ $project_id_file_name ]; then
+      current_project_file=$(jq -n "$project_files" | jq --arg FILE_NAME "$project_id_file_name" -jc '
+        .data | map(select(.fileName<=($FILE_NAME))) | .[0]')
+    else
+      current_project_file=$(jq -n "$project_files" | jq --arg RELEASE_FILTER "$RELEASE_NUMBER_FILTER" -jc '
+        .data | sort_by(.id) | reverse | map(select(.releaseType<=($RELEASE_FILTER|tonumber))) | .[0]')
+    fi
     # Logic to grab the latest release over the entire pagination
     if [ ! "$PROJECT_FILE" ]; then
       PROJECT_FILE=$current_project_file
@@ -143,6 +149,40 @@ downloadModPackfromModFile() {
   fi
 }
 
+downloadDependencies(){
+  if [ "$PROJECT_FILE" ]; then
+    dependencies=$(jq -n "$PROJECT_FILE" | jq -jc '.dependencies' )
+    required_dependencies=$(jq -n "$dependencies" | jq --arg REQUIRED_FILTER "3" -jc '
+      map(select(.relationType==($REQUIRED_FILTER|tonumber)))')
+    if [ "$required_dependencies" ]; then
+      jq -n "$required_dependencies" | jq -c '.[]?' | while read current_dependency; do
+        mod_id=$(jq -n "$current_dependency" | jq -jc '.modId' )
+
+        # BROKEN: Example Voice mod keeps returning the voice mod file id instead of the mod file id.
+        # file_id=$(jq -n "$current_dependency" | jq -jc '.fileId' )
+        # dependency_data=$(curl -X GET -s \
+        #   "${FORGEAPI_BASE_URL}/mods/${mod_id}/files/${file_id}/download-url" \
+        #   -H 'Accept: application/json' -H 'x-api-key: '${MODS_FORGEAPI_KEY}'')  
+        # if [ ! "$dependency_data" ]; then
+        #   log "ERROR: unable to retrieve dependency data files for ${project_id} from ForgeAPI"
+        #   exit 2
+        # fi
+        # dependency_download_url=$(jq -n "$dependency_data" | jq -jc '.data' )
+        # echo "Downloading dependency ${dependency_download_url}"
+        # if ! get -o "${out_dir}/" $dependency_download_url ; then 
+        #   log "ERROR: failed to download dependency from ${dependency_download_url}"
+        #   exit 2
+        # fi
+
+        # Using current mod path and release to go get the REQUIRED DEPENDENCY
+        # NOTE: we are ASUMING it will be release.
+        modFileByProjectID $mod_id "release"
+        downloadModPackfromModFile
+      done
+    fi
+  fi
+}
+
 # Use forge api json file to filter and download the correct mods
 if [ "$MODS_FORGEAPI_FILE" ] && [ -z "$MODS_FORGEAPI_PROJECTIDS" ]; then
   ensureModKey
@@ -150,18 +190,19 @@ if [ "$MODS_FORGEAPI_FILE" ] && [ -z "$MODS_FORGEAPI_PROJECTIDS" ]; then
     log "ERROR: given MODS_FORGEAPI_FILE file does not exist"
     exit 2
   fi
-  MODS_FORGEAPI_PROJECTIDS=$(jq --raw-output '[.[] | .projectId] | join(",")' $MODS_FORGEAPI_FILE)
-  if [ ! "$MODS_FORGEAPI_PROJECTIDS" ]; then
-    log "ERROR: unable to retrieve packs from $MODS_FORGEAPI_FILE"
-    exit 2
-  fi
 
   # Needs loop here to look up release types befor calling download.
-  for project_id in ${MODS_FORGEAPI_PROJECTIDS//,/ }; do
-    current_release_type=$(jq --arg PROJECT_ID "$project_id" -jc '
-      .[] | select(.projectId==$PROJECT_ID) | .releaseType' "$MODS_FORGEAPI_FILE")
-    modFileByProjectID $project_id $current_release_type
-    downloadModPackfromModFile    
+  jq -n "$required_dependencies" | jq -c '.[]?' | while read current_project; do
+    # Per stack overflow we can use //empty to return empty string that works with -z
+    project_id=$(jq -n "$current_project" | jq -jc '.projectId // empty' )
+    current_release_type=$(jq -n "$current_project" | jq -jc '.releaseType // empty' )
+    current_file_name=$(jq -n "$current_project" | jq -jc '.fileName // empty' )
+
+    modFileByProjectID $project_id $current_release_type $current_file_name
+    downloadModPackfromModFile
+    if isTrue "${MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES}"; then
+      downloadDependencies
+    fi
   done
 fi
 
@@ -170,7 +211,10 @@ if [ "$MODS_FORGEAPI_PROJECTIDS" ] && [ -z "$MODS_FORGEAPI_FILE" ]; then
   ensureModKey
   for project_id in ${MODS_FORGEAPI_PROJECTIDS//,/ }; do
     modFileByProjectID $project_id 
-    downloadModPackfromModFile    
+    downloadModPackfromModFile
+    if isTrue "${MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES}"; then
+      downloadDependencies
+    fi
   done
 fi
 

tests/setuponlytests/forgeapimods_file/docker-compose.yml

@@ -8,10 +8,11 @@ services:
       EULA: "TRUE"
       SETUP_ONLY: "TRUE"
       VERSION: ${MINECRAFT_VERSION:-LATEST}
-      MODS_FORGEAPI_FILE: /config/example.json
+      MODS_FORGEAPI_FILE: /config/forgeapi_mods.json
       # Key is defined in .github/workflows/pr.yml and ci.yml
       # This should be coming from github secrets.
       MODS_FORGEAPI_KEY: ${MODS_FORGEAPI_KEY}
       REMOVE_OLD_FORGEAPI_MODS: "TRUE"
+      MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES: "FALSE"
     volumes:
-      - ./example.json:/config/example.json:ro
+      - ./forgeapi_mods.json:/config/forgeapi_mods.json:ro

tests/setuponlytests/forgeapimods_file/forgeapi_mods.json

@@ -7,5 +7,11 @@
         "name": "Fabric Voice Mod",
         "projectId": "416089",
         "releaseType": "beta"
+    },
+    {
+        "name": "Biomes o plenty",
+        "projectId": "220318",
+        "fileName": "BiomesOPlenty-1.18.1-15.0.0.100-universal.jar",
+        "releaseType": "release"
     }
 ]

tests/setuponlytests/forgeapimods_file/require.sh

@@ -0,0 +1 @@
+[[ $MODS_FORGEAPI_KEY ]] || exit 1

tests/setuponlytests/forgeapimods_projectids/docker-compose.yml

@@ -8,7 +8,8 @@ services:
       EULA: "TRUE"
       SETUP_ONLY: "TRUE"
       VERSION: ${MINECRAFT_VERSION:-LATEST}
-      MODS_FORGEAPI_PROJECTIDS: 306612,416089
+      MODS_FORGEAPI_DOWNLOAD_DEPENDENCIES: "TRUE"
+      MODS_FORGEAPI_PROJECTIDS: 306612,416089,220318
       # Allows for Beta releases of 416089 the Fabric Voice Mod 
       MODS_FORGEAPI_RELEASES: BETA
       MODS_FORGEAPI_KEY: ${MODS_FORGEAPI_KEY}

tests/setuponlytests/forgeapimods_projectids/require.sh

@@ -0,0 +1 @@
+[[ $MODS_FORGEAPI_KEY ]] || exit 1

tests/setuponlytests/test.sh

@@ -11,6 +11,17 @@ setupOnlyMinecraftTest(){
   cd "$folder"
   result=0
 
+  if [ -f require.sh ]; then
+    # require.sh scripts can check for environment variables, etc that are required for the test.
+    # The script should exit with a non-zero status to indicate the test requirements are missing
+    # and the test should be skipped
+    if ! bash require.sh; then
+      echo "${folder} SKIP"
+      cd ..
+      return 0
+    fi
+  fi
+
   if ! logs=$(docker-compose run mc 2>&1); then
     echo "${folder} test scenario FAILED"
     echo ":::::::::::: LOGS ::::::::::::::::