feat: add LLM import pipeline with file upload and text input

2026-05-21 10:29:35 -04:00
parent b179135136
commit 578d1ae068
4 changed files with 221 additions and 35 deletions
@@ -83,6 +83,12 @@ main = do
    putStrLn $ "[roux] config file: " <> optConfigFile opts

    -- Load config (file may be missing; LLM features disabled gracefully)
-    _config <- Config.loadConfig (optConfigFile opts)
-    waiApp <- Roux.app (optRecipeDir opts)
+    configResult <- Config.loadConfig (optConfigFile opts)
+    let config = case configResult of
+            Right cfg -> cfg
+            Left _ -> Config.RouxConfig Nothing
+    case configResult of
+        Left err -> putStrLn $ "[roux] warning: " <> err <> " — LLM features disabled"
+        Right _ -> pure ()
+    waiApp <- Roux.app (optRecipeDir opts) config
    Warp.runSettings settings waiApp
@@ -29,6 +29,7 @@ ghc-options:
 dependencies:
  - base >= 4.7 && < 5
  - aeson
+  - base64-bytestring
  - json-fleece-aeson
  - json-fleece-core
  - blaze-html
@@ -42,6 +42,7 @@ library
  build-depends:
      aeson
    , base >=4.7 && <5
+    , base64-bytestring
    , blaze-html
    , blaze-markup
    , bytestring
@@ -79,6 +80,7 @@ executable check-recipes
  build-depends:
      aeson
    , base >=4.7 && <5
+    , base64-bytestring
    , blaze-html
    , blaze-markup
    , bytestring
@@ -117,6 +119,7 @@ executable roux-server
  build-depends:
      aeson
    , base >=4.7 && <5
+    , base64-bytestring
    , blaze-html
    , blaze-markup
    , bytestring
@@ -161,6 +164,7 @@ test-suite roux-server-test
  build-depends:
      aeson
    , base >=4.7 && <5
+    , base64-bytestring
    , blaze-html
    , blaze-markup
    , bytestring
@@ -14,10 +14,11 @@ module Roux.Server (
 import Control.Concurrent (forkIO, threadDelay)
 import Control.Exception (IOException, SomeException, catch, finally, try)
 import Control.Monad (forM_, forever, when)
+import Data.ByteString.Base64 (decode)
 import Data.ByteString.Builder (lazyByteString)
 import Data.ByteString.Lazy (ByteString)
 import qualified Data.ByteString.Lazy as LB
-import Data.Char (digitToInt, toLower)
+import Data.Char (digitToInt, isAlphaNum, toLower)
 import Data.IORef (IORef, atomicModifyIORef', newIORef, readIORef, writeIORef)
 import Data.List (isSuffixOf)
 import Data.Map.Strict (Map)
@@ -29,7 +30,7 @@ import Data.Time.Clock (diffUTCTime, getCurrentTime)

 import qualified Network.HTTP.Types as HTTP
 import qualified Network.Wai as Wai
-import System.Directory (doesFileExist, findExecutable, makeAbsolute)
+import System.Directory (createDirectoryIfMissing, doesFileExist, findExecutable, makeAbsolute, removeFile)
 import System.FSNotify (
    Event (..),
    EventIsDirectory (..),
@@ -45,6 +46,7 @@ import qualified Data.Aeson as A
 import qualified Data.ByteString as BS
 import Data.Maybe (mapMaybe)
 import qualified Data.Text.Encoding as TE
+import Roux.Config (AnthropicConfig (..), RouxConfig (..))
 import qualified Roux.CooklangPrint as CooklangPrint
 import qualified Roux.Html as Html
 import qualified Roux.RecipeIndex as Idx
@@ -65,12 +67,12 @@ readRequestBody req = do
            then pure (reverse acc)
            else gather (chunk : acc)

-{- | Build the WAI 'Application' given a path to the recipe directory.
+{- | Build the WAI 'Application' given a path to the recipe directory and config.

 Recipes are scanned at startup and then watched live via fsnotify.
 -}
-app :: FilePath -> IO Wai.Application
-app recipeDir = do
+app :: FilePath -> RouxConfig -> IO Wai.Application
+app recipeDir config = do
    putStrLn $ "[roux] scanning recipes from " <> recipeDir
    recipes <- Idx.scanRecipes recipeDir
    let count = length recipes
@@ -92,7 +94,7 @@ app recipeDir = do
        forkIO $
            watchRecipes absDir state changeLog `catch` \e ->
                putStrLn $ "[roux] watcher thread crashed: " <> show (e :: SomeException)
-    pure (router absDir state changeLog)
+    pure (router absDir config state changeLog)
  where
    isRight (Right _) = True
    isRight (Left _) = False
@@ -174,19 +176,21 @@ sseHandler changeLog _request respond = do
            go `finally` pure ()

 -- | Route requests — SSE endpoint, recipe pages, or import.
-router :: FilePath -> IORef (Map FilePath Idx.RecipeInfo) -> ChangeLog -> Wai.Application
-router recipeDir state changeLog request respond =
+router :: FilePath -> RouxConfig -> IORef (Map FilePath Idx.RecipeInfo) -> ChangeLog -> Wai.Application
+router recipeDir config state changeLog request respond =
    case Wai.pathInfo request of
        ["events"] -> sseHandler changeLog request respond
-        ["import"] -> importHandler recipeDir state request respond
+        ["import"] -> importHandler recipeDir config state request respond
        _ -> handleRequest state request respond

 {- | Handle GET and POST requests to /import.
 GET  — show the import form.
-POST — scrape the URL, convert to Cooklang, save the file.
+POST — dispatch based on input: URL (existing scraper), text, or file (Anthropic).
+Files are uploaded as base64-encoded content via hidden form fields
+(file-b64, file-name, file-mime) set by client-side JavaScript.
 -}
-importHandler :: FilePath -> IORef (Map FilePath Idx.RecipeInfo) -> Wai.Application
-importHandler recipeDir _state request respond =
+importHandler :: FilePath -> RouxConfig -> IORef (Map FilePath Idx.RecipeInfo) -> Wai.Application
+importHandler recipeDir config _state request respond =
    case Wai.requestMethod request of
        "GET" ->
            respond (htmlResponse (Html.importPage Nothing))
@@ -194,27 +198,47 @@ importHandler recipeDir _state request respond =
            body <- readRequestBody request
            let params = parseFormBody body
                url = maybe "" id (lookup "url" params)
-            if T.null url
-                then respond (htmlResponse (Html.importPage (Just ("URL is required" :: Text))))
-                else do
-                    putStrLn $ "[roux] import request for " <> T.unpack url
-                    result <- try $ runImportPipeline recipeDir url
-                    case result of
-                        Left (e :: SomeException) ->
-                            respond $
-                                htmlResponse $
-                                    Html.importResultPage $
-                                        Html.ImportError ("Unexpected error: " <> T.pack (show e))
-                        Right (Left err) ->
-                            respond $
-                                htmlResponse $
-                                    Html.importResultPage err
-                        Right (Right filename) ->
-                            respond $
-                                Wai.responseLBS
-                                    HTTP.status303
-                                    [("Location", "/recipes/" <> TE.encodeUtf8 (T.pack filename))]
-                                    ""
+                text = maybe "" id (lookup "text" params)
+                fileB64 = maybe "" id (lookup "file-b64" params)
+                fileName = maybe "" id (lookup "file-name" params)
+                fileMime = maybe "" id (lookup "file-mime" params)
+                hasFile = not (T.null fileB64)
+
+            putStrLn $
+                "[roux] import request: url="
+                    <> T.unpack url
+                    <> " text="
+                    <> show (not (T.null text))
+                    <> " file="
+                    <> show hasFile
+
+            result <- try $ case (hasFile, not (T.null text), not (T.null url)) of
+                -- File takes priority, then text, then URL
+                (True, _, _) ->
+                    runFileImport recipeDir config fileName fileMime fileB64
+                (_, True, _) ->
+                    runTextImport recipeDir config text
+                (_, _, True) ->
+                    runImportPipeline recipeDir url
+                _ ->
+                    pure (Left (Html.ImportError "Provide a URL, recipe text, or file to import."))
+
+            case result of
+                Left (e :: SomeException) ->
+                    respond $
+                        htmlResponse $
+                            Html.importResultPage $
+                                Html.ImportError ("Unexpected error: " <> T.pack (show e))
+                Right (Left err) ->
+                    respond $
+                        htmlResponse $
+                            Html.importResultPage err
+                Right (Right filename) ->
+                    respond $
+                        Wai.responseLBS
+                            HTTP.status303
+                            [("Location", "/recipes/" <> TE.encodeUtf8 (T.pack filename))]
+                            ""
        _ -> respond notFound

 -- | Parse URL-encoded form body into key-value pairs.
@@ -336,6 +360,157 @@ runImportPipeline recipeDir url = do
                            putStrLn $ "[roux] successfully imported " <> T.unpack url <> " as " <> filename
                            return $ Right filename

+{- | Import a recipe from an uploaded file via the LLM pipeline.
+The file content is base64-encoded in the form data.
+-}
+runFileImport ::
+    -- | recipe directory
+    FilePath ->
+    -- | server config (for Anthropic settings)
+    RouxConfig ->
+    -- | original filename
+    Text ->
+    -- | MIME type
+    Text ->
+    -- | base64-encoded file content
+    Text ->
+    IO (Either Html.ImportError FilePath)
+runFileImport recipeDir config originalName _mimeType b64Text = do
+    -- Decode base64
+    case decode (TE.encodeUtf8 b64Text) of
+        Left err ->
+            pure $
+                Left $
+                    Html.ImportError $
+                        "Could not decode uploaded file: " <> T.pack err
+        Right content -> do
+            -- Ensure recipe dir exists
+            createDirectoryIfMissing True recipeDir
+            -- Save the original file
+            let sourceFilename = map (\c -> if isAlphaNum c || c == '.' || c == '-' || c == '_' then c else '_') (T.unpack originalName)
+                sourcePath = recipeDir </> sourceFilename
+            BS.writeFile sourcePath content
+            putStrLn $ "[roux] saved uploaded file to " <> sourcePath
+            -- Call the LLM pipeline with the saved file
+            case rcAnthropic config of
+                Nothing ->
+                    pure $
+                        Left $
+                            Html.ImportError $
+                                "LLM import is not configured. Add an anthropic.api_key to the config file."
+                Just anthropicConfig -> do
+                    result <- runLLMPipeline recipeDir anthropicConfig (Just sourcePath) ""
+                    case result of
+                        Right filename -> pure $ Right filename
+                        Left err -> pure $ Left err
+
+-- | Import a recipe from pasted text via the LLM pipeline.
+runTextImport ::
+    FilePath ->
+    RouxConfig ->
+    Text ->
+    IO (Either Html.ImportError FilePath)
+runTextImport recipeDir config text = do
+    case rcAnthropic config of
+        Nothing ->
+            pure $
+                Left $
+                    Html.ImportError $
+                        "LLM import is not configured. Add an anthropic.api_key to the config file."
+        Just anthropicConfig ->
+            runLLMPipeline recipeDir anthropicConfig Nothing text
+
+-- | Run the full LLM import pipeline: call convert-to-cooklang -> parse -> save.
+runLLMPipeline ::
+    -- | recipe directory
+    FilePath ->
+    -- | API config
+    AnthropicConfig ->
+    -- | Just path = uploaded source file to keep
+    Maybe FilePath ->
+    -- | recipe text (or empty if file)
+    Text ->
+    IO (Either Html.ImportError FilePath)
+runLLMPipeline recipeDir anthropicConfig mSourceFile text = do
+    -- Step 1: locate the script
+    script <- findConvertScript
+    -- Step 2: write a temp config for the subprocess
+    let configPath = recipeDir </> ".convert-config.yaml"
+    writeConvertConfig configPath anthropicConfig
+    -- Step 3: build subprocess args
+    let args = case mSourceFile of
+            Just fp -> ["--config", configPath, "--file", fp]
+            Nothing -> ["--config", configPath, "--text", T.unpack text]
+    putStrLn $ "[roux] running " <> script <> " for LLM import"
+    (exitCode, stdoutBytes, stderrBytes) <-
+        readProcessBytes script args
+    -- Clean up temp config
+    _ <- try (removeFile configPath) :: IO (Either IOException ())
+    case exitCode of
+        ExitFailure _ -> do
+            let errMsg = TE.decodeUtf8 (LB.toStrict stderrBytes)
+            putStrLn $ "[roux] convert-to-cooklang failed: " <> T.unpack errMsg
+            return $
+                Left $
+                    Html.ImportError $
+                        "Could not convert recipe: " <> errMsg
+        ExitSuccess -> do
+            -- Step 4: parse the Cooklang output to extract the title
+            let cooklangText = TE.decodeUtf8 (LB.toStrict stdoutBytes)
+            case parseRecipeTitle cooklangText of
+                Nothing -> do
+                    putStrLn "[roux] could not extract title from Cooklang output"
+                    return $
+                        Left $
+                            Html.ImportError $
+                                "Could not determine recipe title from AI output."
+                Just title -> do
+                    let filename = CooklangPrint.filenameFromTitle title
+                        filepath = recipeDir </> filename
+                    -- Step 5: write the .cook file
+                    putStrLn $ "[roux] writing imported recipe to " <> filepath
+                    LB.writeFile filepath stdoutBytes
+                    putStrLn $ "[roux] successfully imported recipe as " <> filename
+                    return $ Right filename
+
+-- | Locate the convert-to-cooklang script.
+findConvertScript :: IO FilePath
+findConvertScript = do
+    mPath <- findExecutable "convert-to-cooklang"
+    case mPath of
+        Just p -> pure p
+        Nothing -> pure "scripts/convert-to-cooklang"
+
+-- | Write a temporary YAML config for the subprocess.
+writeConvertConfig :: FilePath -> AnthropicConfig -> IO ()
+writeConvertConfig path ac = do
+    let pairs =
+            [ "anthropic:"
+            , "  api_key: " <> T.unpack (acApiKey ac)
+            ]
+                ++ case acBaseUrl ac of
+                    Just url -> ["  base_url: " <> T.unpack url]
+                    Nothing -> []
+                ++ case acModel ac of
+                    Just m -> ["  model: " <> T.unpack m]
+                    Nothing -> []
+    writeFile path (unlines pairs)
+
+{- | Parse the recipe title from Cooklang front matter.
+Looks for "title: ..." in the YAML front matter block.
+-}
+parseRecipeTitle :: Text -> Maybe Text
+parseRecipeTitle cooklang =
+    case T.breakOn "---" cooklang of
+        (_, afterFirst) ->
+            let (_separator, inside) = T.breakOn "---" (T.drop 3 afterFirst)
+                frontMatter = T.take (T.length inside) (T.drop 3 afterFirst)
+             in case T.breakOn "title:" frontMatter of
+                    (_, afterTitle) ->
+                        let titleLine = T.takeWhile (/= '\n') (T.drop 6 afterTitle)
+                            cleaned = T.strip titleLine
+                         in if T.null cleaned then Nothing else Just cleaned
+
 {- | Watch a directory for .cook file changes and update the shared state.
 Uses watchDir callback for simplicity.
 -}