Issue 2756:Clean up interface translation files

tools/generateUnusedKeys

@@ -16,11 +16,14 @@
 #   FILEBROWSER_POPUPRANK
 #   FILEBROWSER_POPUPCOLORLABEL
 #
+# The script can also clean up already-translated files by removing keys which
+# do not exist in default.
+#
 # Run the script from the project root:
 #   ./tools/generateUnusedKeys
 #
 # Doublecheck the deletion before committing.
-# Run ./tools/generateTranslationDiffs after running this script.
+# Run ./tools/generateTranslationDiffs before and after running this script.
 #
 # Blame DrSlony
 # Please report bugs or enhancements to http://code.google.com/p/rawtherapee/issues/list
@@ -31,7 +34,7 @@ if [[ -w $tmp ]]; then
 fi
 
 abort () {
-  printf "%s\n" "" "Aborted" "Removing leftover files:"
+  printf '%s\n' "" "Aborted" "Removing leftover files:"
   [[ -e "$tmp" ]] && rm "$tmp"
   rm -v --interactive=once sed*
   exit 1
@@ -39,7 +42,7 @@ abort () {
 
 trap 'abort' HUP INT QUIT ABRT TERM
 
-cd "rtdata/languages" || { printf "%s\n" "You must run this script from the root of the project."; exit 1; }
+cd "rtdata/languages" || { printf '%s\n' "You must run this script from the root of the project."; exit 1; }
 # Build array of all interface translation files, or use user-specified ones only
 unset langFiles
 if [[ $# = 0 ]]; then
@@ -50,7 +53,7 @@ else
   langFiles=("$@")
   for langFile in "${langFiles[@]}"; do
     if [[ ! -w $langFile ]]; then
-      printf "%s\n" "File \"$langFile\" not found or not writable." ""
+      printf '%s\n' "File \"$langFile\" not found or not writable." ""
       exit 1
     fi
   done
@@ -59,12 +62,12 @@ fi
 dos2unix default 2>/dev/null
 
 t1="$(date +%s)"
-printf "%s\n" 'Matching keys in "default" against .cc and .h files' 'Unmatched keys follow:'
+printf '%s\n' 'Matching keys in "default" against .cc and .h files' 'Unmatched keys follow:'
 unset delLines
 while read -r 'defLine'; do
   grep -Ir --include=\*.{cc,h} --exclude-dir="klt" "${defLine%%;*}" ../../* | grep -Ev "//.*${defLine%%;*}|/\*.*${defLine%%;*}|${defLine%%;*}.*\*/" &>/dev/null
   if [[ $? = 1 ]]; then
-    printf "  %s\n" "${defLine%%;*}"
+    printf '  %s\n' "${defLine%%;*}"
     delLines+=("${defLine%%;*}")
   fi
 done < <(grep -Ev "^(#|$)|HISTORY_MSG_" "default" | sed -e "s/EXTPROGTARGET_[0-9]*/EXTPROGTARGET_/" -e "s/FILEBROWSER_POPUPCOLORLABEL[0-9]*/FILEBROWSER_POPUPCOLORLABEL/" -e "s/FILEBROWSER_POPUPRANK[0-9]*/FILEBROWSER_POPUPRANK/" | sort -Vu)
@@ -78,26 +81,25 @@ done < <(grep -Ev "^(#|$)|HISTORY_MSG_" "default" | sed -e "s/EXTPROGTARGET_[0-9
 # Dynamically built keys like HISTORY_MSG_1 can't be grepped in the code,
 # so it renames KEY_1-KEY_9 to KEY_ so that they can be grepped and therefore ignored.
 # The piped grep in the loop
-
 t2="$(date +%s)"
 tt=$((t2-t1))
-printf "%s\n" "" "Scan took $tt seconds" "" "Double-checking the code for matched keys:"
+printf '%s\n' "" "Scan took $tt seconds" "" "Double-checking the code for matched keys:"
 
 for delLine in "${delLines[@]}"; do
-  printf "%s\n" "$delLine"
+  printf '%s\n' "$delLine"
   grep -Ir --include=\*.{cc,h} --exclude-dir="klt" "${delLine}" ../../*
 done
 echo
 
 read -r -p 'Write results to "unmatched"? [y/n] '
 if [[ $REPLY = y || $REPLY = Y ]]; then
-  printf "%s\n" "${delLines[@]}" > unmatched
+  printf '%s\n' "${delLines[@]}" > unmatched
-  printf "%s\n" ""
 fi
+printf '%s\n' ""
 
-read -r -p "Delete keys from all ${#langFiles[@]} interface language files? [y/n] "
+read -r -p "Delete ${#delLines[@]} keys from all ${#langFiles[@]} interface language files? [y/n] "
 if [[ $REPLY = y || $REPLY = Y ]]; then
-  printf "%s\n" "Removing keys from:"
+  printf '%s\n' "Removing keys from:"
   i=1
   ttot1="$(date +%s)"
   for file in "${langFiles[@]}"; do
@@ -108,12 +110,52 @@ if [[ $REPLY = y || $REPLY = Y ]]; then
     done
     t2="$(date +%s)"
     tt=$((t2-t1))
-    printf "%s\n" " - took $tt seconds"
+    printf '%s\n' " - took $tt seconds"
     ((i++))
   done
   ttot2="$(date +%s)"
   ttot=$((ttot2-ttot1))
   tsec=$((ttot%60))
   tmin=$((ttot/60))
-  printf "%s\n" "Finished updating ${#langFiles[@]} files." "Total time: ${tmin}m ${tsec}s"
+  printf '%s\n' "Finished updating ${#langFiles[@]} files." "Total time: ${tmin}m ${tsec}s" ""
+
+  printf '%s\n' "The above cleaned up \"default\" from keys not found in the code and then it removed the same keys from the translation files. However if the translation files contain keys which were not in \"default\" then they would remain (because it would take too long to scan the source code for each key in each translation). Since at this point \"default\" is good as gold, it's most time efficient then to match each interface translation file against it and remove all keys not found in it." "" | fold -s
+  read -r -p "Match ${#langFiles[@]} interface translation files against \"default\" and delete unmatched keys? [y/n] "
+  if [[ $REPLY = y || $REPLY = Y ]]; then
+    printf '%s\n' "Removing keys from:"
+    i=1
+    ttot1="$(date +%s)"
+    for file in "${langFiles[@]}"; do
+      printf "%02d - ${file#.*/}" "$i"
+      t1="$(date +%s)"
+      unset delLines
+      # Read line by line
+      while read -r line; do
+        # If line starts with a real key, not a comment or empty line
+        if [[ $line =~ ^[A-Z0-9_]+\; ]]; then
+          # then get the key part
+          key="${line%%;*}"
+          # quietly check whether it exists in default
+          grep -Irq "$key" default
+          # and if match fails, put it on the kill list
+          if [[ $? = 1 ]]; then
+            delLines+=("${key}")
+          fi
+        fi
+      done < "$file"
+      # kill szeva
+      for key in "${delLines[@]}"; do
+        sed -i "/.\?$key/d" "$file"
+      done
+      t2="$(date +%s)"
+      tt=$((t2-t1))
+      printf '%s\n' " - took $tt seconds"
+    done
+    ttot2="$(date +%s)"
+    ttot=$((ttot2-ttot1))
+    tsec=$((ttot%60))
+    tmin=$((ttot/60))
+    printf '%s\n' "Finished updating ${#langFiles[@]} files." "Total time: ${tmin}m ${tsec}s"
+  fi
+
 fi