feat: add support for custom transformers-specific generation tag in template processing

Author: CodeWithKyrian

src/Core/Lexer.php

@@ -40,6 +40,10 @@ public static function preprocess(string $template, bool $lstripBlocks = false,
         $template = preg_replace('/-#}\s*/', '#}', $template);
         $template = preg_replace('/\s*{#-/', '{#', $template);
 
+        // Handle the custom transformers-specific `generation` tag.
+        // See https://github.com/huggingface/transformers/pull/30650 for more information.
+        $template = preg_replace('/{%\s*(end)?generation\s*%}/', '', $template);
+
         return $template;
     }