Added CI that calculates the progress on Languagepacks (#701)

* Added script and CI that calculates the progress on Languagepacks
* Improved error handling and added Workflow summary

.github/scripts/localizationChecker.py

@@ -0,0 +1,111 @@
+import lupa.lua54 as lupa
+from lupa.lua54 import LuaRuntime
+
+from pathlib import Path
+
+lua = LuaRuntime(unpack_returned_tuples=True)
+import glob, os
+SCRIPT_PATH = os.path.realpath(__file__)
+SOURCE_MOD_DIRECTORY = os.path.dirname(SCRIPT_PATH)+"/../.."
+
+if "GITHUB_WORKSPACE" in os.environ:
+    SOURCE_MOD_DIRECTORY = os.environ["GITHUB_WORKSPACE"]
+
+# Reset
+Color_Off='\033[0m'       # Text Reset
+
+# Regular Colors
+Black='\033[0;30m'        # Black
+Red='\033[0;31m'          # Red
+Green='\033[0;32m'        # Green
+Yellow='\033[0;33m'       # Yellow
+Blue='\033[0;34m'         # Blue
+Purple='\033[0;35m'       # Purple
+Cyan='\033[0;36m'         # Cyan
+White='\033[0;37m'        # White
+BWhite='\033[1;37m'       # Bold White
+
+ignoreNodesWithName = {"fonts"}
+
+maxChecklimit = {"tarotClothBuffs": 2}
+
+
+# count en_us entries for stats
+def count_entries(t):
+    count = 0
+    for k in t:
+        count += 1
+        if lupa.lua_type(t[k]) == "table":
+            count += count_entries(t[k])
+    return count
+
+def getMaxCheckLimit(parentName):
+    for entry in maxChecklimit:
+        if entry in parentName:
+            return maxChecklimit[entry]
+    return float('inf')
+
+# find entries in table2 that are missing or different than in table 1
+def compare_tables(table1, table2, prev_key):
+    errorCount = 0
+    checkLimit = getMaxCheckLimit(prev_key)
+    for k in table1:
+        checkLimit -= 1
+        if k in ignoreNodesWithName or checkLimit < 0:
+            # dont evaluate nodes that are listed in this table
+            continue
+        if k not in table2:
+            print(f"\tTable '{prev_key}' does not contain key: {k}")
+            errorCount += 1
+            # table is missing. add all missing entries as error
+            if lupa.lua_type(table1[k]) == "table":
+                errorCount += count_entries(table1[k])
+        elif lupa.lua_type(table2[k]) != lupa.lua_type(table1[k]):
+            print(f"\tType mismatch in table '{prev_key}', key: {k}")
+            errorCount += 1
+        elif lupa.lua_type(table2[k]) == "table":
+            errorCount += compare_tables(table1[k], table2[k], str(prev_key)+"->"+str(k))
+    return errorCount
+
+
+lua.execute('EID = {}; EID.descriptions = {} function EID:updateDescriptionsViaTable(changeTable, tableToUpdate) for k,v in pairs(changeTable) do if v == "" then tableToUpdate[k] = nil else tableToUpdate[k] = v end	end end')
+g = lua.globals()
+
+
+print("reading: en_us.lua")
+for englishFile in glob.glob(SOURCE_MOD_DIRECTORY+"/**/en_us.lua", recursive=True):
+    lua.execute(open(englishFile, "r", encoding="UTF-8").read())
+
+en_us_entries = count_entries(g.EID['descriptions']['en_us'])
+print("en_us entries:", en_us_entries)
+
+
+languages = {}
+for file in glob.glob(SOURCE_MOD_DIRECTORY+"/**/ab+/*.lua", recursive=True) + glob.glob(SOURCE_MOD_DIRECTORY+"/**/rep/*.lua", recursive=True):
+    if "en_us" not in file and "transformations" not in file:
+        print("reading:",file)
+        lua.execute(open(file, "r", encoding="UTF-8").read())
+        languages[os.path.basename(file).replace(".lua","")] = 0
+
+for lang in languages:
+    print(f"Evaluating language '{lang}'..")
+    errorCount = compare_tables(g.EID['descriptions']['en_us'], g.EID['descriptions'][lang], lang)
+    percentage = (en_us_entries - errorCount) / en_us_entries*100
+    languages[lang] = [percentage, errorCount]
+    print(f"{Red}Errors found: {errorCount} / {en_us_entries}{Color_Off}\n\n")
+
+
+
+gitWorkflowSummary = "### Translation progress\n| Language | Completion | Missing entries |\n|---|---|---|\n"
+print(f"{Blue}Translation progress:{Color_Off}")
+for lang in languages:
+    print(
+        f"\t{BWhite}{lang}{Color_Off}\t{Blue}{round(languages[lang][0],2)}%{Color_Off}\t{Red}{languages[lang][1]} missing{Color_Off}"
+    )
+    errorMessage = languages[lang][1] if languages[lang][1] >0 else "🎉"
+    gitWorkflowSummary += f"| {lang} | {round(languages[lang][0],2)}% | {errorMessage} |\n"
+
+
+if "GITHUB_STEP_SUMMARY" in os.environ:
+    print("\n\nwrite workflow summary to: ", os.environ["GITHUB_STEP_SUMMARY"])
+    Path(os.environ["GITHUB_STEP_SUMMARY"]).write_text(gitWorkflowSummary)

.github/workflows/localization.yml

@@ -0,0 +1,22 @@
+name: Check Localization
+
+on: [push, workflow_dispatch]
+
+jobs:
+  localizationcheck:
+    name: Check Localization process
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4.5.0
+        with:
+          python-version: '3.10'
+
+      - name: Install PIP packages
+        run: |
+              pip install lupa
+
+      - name: Calculate Localization progress
+        run: python .github/scripts/localizationChecker.py

features/eid_debugging.lua

@@ -19,39 +19,61 @@ EID:countEntries(EID.descriptions["en_us"])
 local enUSEntries = count
 print("en_us entries: "..enUSEntries)
 
-for _,lang in ipairs(languageFilesToCheck) do
+
+local maxChecklimit = {["tarotClothBuffs"] = 2}
+function EID:getMaxCheckLimit(parentName)
+    for entry, _ in pairs(maxChecklimit) do
+        if string.find(parentName, entry) then
+            return maxChecklimit[entry]
+		end
+	end
+    return math.maxinteger
+end
+
+local ignoreNodesWithName = {["fonts"] = true}
+
+
+for _, lang in ipairs(languageFilesToCheck) do
 	print("Now checking integrity of languagefile: " .. lang)
 	-- Generic function to compare two tables
 	function EID:compareTables(table1, table2, prevKey, progress)
+		local checkLimit = EID:getMaxCheckLimit(prevKey)
 		for k, _ in pairs(table1) do
 			progress[1] = progress[1] + 1
-			if not table2[k] then
+			checkLimit = checkLimit - 1
-				print(" Table '" .. prevKey .. "' does not contain key: " .. k)
+			if not (ignoreNodesWithName[k] or checkLimit < 0) then
-				progress[2] = progress[2] + 1
+				-- only evaluate nodes that are not listed in this table
-			elseif type(table2[k]) ~= type(table1[k]) then
+				if not table2[k] then
-				print("Type mismatch in table '" .. prevKey .. "', key: " .. k)
+					print(" Table '" .. prevKey .. "' does not contain key: " .. k)
-				progress[2] = progress[2] + 1
+					progress[2] = progress[2] + 1
-			elseif type(table2[k]) == "table" then
+				elseif type(table2[k]) ~= type(table1[k]) then
-				EID:compareTables(table1[k], table2[k], prevKey.."->"..k, progress)
+					print("Type mismatch in table '" .. prevKey .. "', key: " .. k)
-			else
+					progress[2] = progress[2] + 1
-				-- check for broken markup stuff
+				elseif type(table2[k]) == "table" then
-				local filteredText = EID:replaceShortMarkupStrings(table2[k])
+				  EID:compareTables(table1[k], table2[k], prevKey.."->"..k, progress)
-				local textPartsTable = EID:filterColorMarkup(filteredText, EID:getNameColor())
+				else
-				for _, textPart in ipairs(textPartsTable) do
+					-- check for broken markup stuff
-					local filteredSpriteText, spriteTable = EID:filterIconMarkup(textPart[1])
+					local filteredText = EID:replaceShortMarkupStrings(table2[k])
+					local textPartsTable = EID:filterColorMarkup(filteredText, EID:getNameColor())
+					for _, textPart in ipairs(textPartsTable) do
+						local filteredSpriteText, spriteTable = EID:filterIconMarkup(textPart[1])
 
-					if string.find(filteredSpriteText, "{{") or string.find(filteredSpriteText, "}}") then
+						if string.find(filteredSpriteText, "{{") or string.find(filteredSpriteText, "}}") then
-						print(" Table '" .. prevKey .. "' entry '" .. k .. "' does contain a broken markup object: '" .. table2[k])
+							print(" Table '" ..
-						progress[1] = progress[1] - 2
+							prevKey .. "' entry '" .. k .. "' does contain a broken markup object: '" .. table2[k])
-						progress[2] = progress[2] + 1
+							progress[1] = progress[1] - 2
-						break
+							progress[2] = progress[2] + 1
-					else
+							break
-						for _, sprite in ipairs(spriteTable) do
+						else
-							if sprite[1][1] == "ERROR" then
+							for _, sprite in ipairs(spriteTable) do
-								print(" Table '" .. prevKey .. "' entry '" .. k .. "' does contain a bad icon markup in string: '" .. table2[k])
+								if sprite[1][1] == "ERROR" then
-								progress[1] = progress[1] - 2
+									print(" Table '" ..
-								progress[2] = progress[2] + 1
+									prevKey ..
-								break
+									"' entry '" .. k .. "' does contain a bad icon markup in string: '" .. table2[k])
+									progress[1] = progress[1] - 2
+									progress[2] = progress[2] + 1
+									break
+								end
 							end
 						end
 					end