Improve translation generation

src/Appwrite/Platform/Tasks/DevGenerateTranslations.php

@@ -24,12 +24,15 @@ class DevGenerateTranslations extends Action
             ->desc('Generate translations in all languages')
             ->param('dry-run', 'true', new Boolean(true), 'If action should do a dry run. Dry run does not write into files', true)
             ->param('api-key', '', new Text(256), 'Open AI API key. Only used during non-dry runs to generate translations.', true)
-            ->callback(fn ($dryRun, $apiKey) => $this->action($dryRun, $apiKey));
+            ->param('language', '', new Text(256), 'Specific language to translate. If left empty, all languages will be done.', true)
+            ->param('all', 'false', new Boolean(true), 'If action should generate all keys in non-english translation files.', true)
+            ->callback(fn ($dryRun, $apiKey, $language, $all) => $this->action($dryRun, $apiKey, $language, $all));
     }
 
-    public function action(bool|string $dryRun, string $apiKey): void
+    public function action(mixed $dryRun, string $apiKey, string $language, mixed $all): void
     {
         $dryRun = \strval($dryRun) === 'true';
+        $all = \strval($all) === 'true';
 
         Console::info("Started");
 
@@ -49,6 +52,12 @@ class DevGenerateTranslations extends Action
         $files = array_diff(scandir($dir), array('.', '..', $mainFile));
 
         foreach ($files as $file) {
+            if(!empty($language)) {
+                if($file !== $language . '.json') {
+                    continue;
+                }
+            }
+
             $fileJson = \json_decode(\file_get_contents($dir . '/' . $file), true);
             $fileKeys = \array_keys($fileJson);
 
@@ -58,19 +67,47 @@ class DevGenerateTranslations extends Action
             // \file_put_contents($dir . '/' . $file, \json_encode($json, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | 0));
             // continue;
 
-            foreach ($mainKeys as $key) {
-                if (!(\in_array($key, $fileKeys))) {
-                    if ($dryRun) {
-                        Console::warning("{$file} missing translation for {$key}");
-                    } else {
-                        $language = \explode('.', $file)[0];
-                        $translation = $this->generateTranslation($language, $mainJson[$key]);
+            $missingKeys = [];
 
+            foreach ($mainKeys as $key) {
+                if($all) {
+                    $missingKeys[] = $key;
+                } else {
+                    if (!(\in_array($key, $fileKeys))) {
+                        $missingKeys[] = $key;
+                    }
+                }
+            }
+
+            if (\count($missingKeys) > 0) {
+                if ($dryRun) {
+                    $keys = \implode(', ', $missingKeys);
+                    Console::warning("{$file} missing translation for: {$keys}");
+                } else {
+                    $language = \explode('.', $file)[0];
+
+                    foreach ($missingKeys as $missingKey) {
                         $json = \json_decode(\file_get_contents($dir . '/' . $file), true);
-                        $json[$key] = $translation;
+
+                        $translation = $this->generateTranslationHuggingFace($language, $mainJson[$missingKey]);
+
+                        if($all) {
+                            $json[$missingKey] = $translation;
+                        } else {
+                            // This puts new key at beginning to prevent merge conflict issue and ending comma
+                            $newPair = [];
+                            $newPair[$missingKey] = $translation;
+
+                            if(isset($json[$missingKey])) {
+                                unset($json[$missingKey]);
+                            }
+
+                            $json = \array_merge($newPair, $json);   
+                        }
+
                         \file_put_contents($dir . '/' . $file, \json_encode($json, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | 0));
 
-                        Console::success("Generated {$key} for {$language}");
+                        Console::success("Generated {$missingKey} for {$language}");
                     }
                 }
             }
@@ -79,23 +116,25 @@ class DevGenerateTranslations extends Action
         Console::info("Done");
     }
 
-    private function generateTranslation(string $targetLanguage, string $enTranslation): string
+    private function generateTranslationHuggingFace(string $targetLanguage, string $enTranslation): string
     {
-        $response = Client::fetch('https://api.openai.com/v1/chat/completions', [
+        $placeholders = [];
+
+        $id = 0;
+        $pattern = '/{{\w+}}/';
+
+        $enTranslation = preg_replace_callback($pattern, function ($match) use (&$id, &$placeholders) {
+            $placeholders[$id] = $match[0];
+            $key = "<m id={$id} />";
+            $id++;
+            return $key;
+        }, $enTranslation);
+
+        $response = Client::fetch('https://eqiq6qexj1g813kr.us-east-1.aws.endpoints.huggingface.cloud', [
             'content-type' => Client::CONTENT_TYPE_APPLICATION_JSON,
             'Authorization' => 'Bearer ' . $this->apiKey
         ], Client::METHOD_POST, [
-            'model' => 'gpt-4-1106-preview', // https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
-            'messages' => [
-                [
-                    'role' => 'system',
-                    'content' => "Please translate the message user provides from English language to target language. Target language is language of country with country code {$targetLanguage}. Do not translate text inside {{ and }} placeholders. Provide only translated text."
-                ],
-                [
-                    'role' => 'user',
-                    'content' => $enTranslation
-                ]
-            ]
+            'inputs' => '<2' . $targetLanguage . '> ' . $enTranslation,
         ], [], 60);
 
         $body = \json_decode($response->getBody(), true);
@@ -104,6 +143,53 @@ class DevGenerateTranslations extends Action
             throw new Exception($response->getBody() . ' with status code ' . $response->getStatusCode() . ' for language ' . $targetLanguage . ' and message ' . $enTranslation);
         }
 
-        return $body['choices'][0]['message']['content'];
+        $targetTranslation = $body[0]['generated_text'];
+
+        $id = 0;
+        foreach ($placeholders as $placeholder) {
+            $targetTranslation = \str_replace("<m id={$id} />", $placeholder, $targetTranslation);
+            $id++;
+        }
+
+        return $targetTranslation;
+    }
+
+    private function generateTranslationDeepL(string $targetLanguage, string $enTranslation): string
+    {
+        $placeholders = [];
+
+        $id = 0;
+        $pattern = '/{{\w+}}/';
+
+        $enTranslation = preg_replace_callback($pattern, function ($match) use (&$id, &$placeholders) {
+            $placeholders[$id] = $match[0];
+            $key = "<m id={$id} />";
+            $id++;
+            return $key;
+        }, $enTranslation);
+
+        $response = Client::fetch('https://api-free.deepl.com/v2/translate', [
+            'content-type' => Client::CONTENT_TYPE_APPLICATION_JSON,
+            'Authorization' => 'DeepL-Auth-Key ' . $this->apiKey
+        ], Client::METHOD_POST, [
+            'target_lang' => $targetLanguage,
+            'text' => [$enTranslation]
+        ], [], 60);
+
+        $body = \json_decode($response->getBody(), true);
+
+        if ($response->getStatusCode() >= 400) {
+            throw new Exception($response->getBody() . ' with status code ' . $response->getStatusCode() . ' for language ' . $targetLanguage . ' and message ' . $enTranslation);
+        }
+
+        $targetTranslation = $body['translations'][0]['text'];
+
+        $id = 0;
+        foreach ($placeholders as $placeholder) {
+            $targetTranslation = \str_replace("<m id={$id} />", $placeholder, $targetTranslation);
+            $id++;
+        }
+
+        return $targetTranslation;
     }
 }