From 578d1ae068bb8612bb9f1c86d9fd3347268d458f Mon Sep 17 00:00:00 2001 From: James Brechtel Date: Thu, 21 May 2026 10:29:35 -0400 Subject: [PATCH] feat: add LLM import pipeline with file upload and text input --- app/Main.hs | 10 +- package.yaml | 1 + roux-server.cabal | 4 + src/Roux/Server.hs | 241 ++++++++++++++++++++++++++++++++++++++------- 4 files changed, 221 insertions(+), 35 deletions(-) diff --git a/app/Main.hs b/app/Main.hs index bcba47c..ef0a579 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -83,6 +83,12 @@ main = do putStrLn $ "[roux] config file: " <> optConfigFile opts -- Load config (file may be missing; LLM features disabled gracefully) - _config <- Config.loadConfig (optConfigFile opts) - waiApp <- Roux.app (optRecipeDir opts) + configResult <- Config.loadConfig (optConfigFile opts) + let config = case configResult of + Right cfg -> cfg + Left _ -> Config.RouxConfig Nothing + case configResult of + Left err -> putStrLn $ "[roux] warning: " <> err <> " — LLM features disabled" + Right _ -> pure () + waiApp <- Roux.app (optRecipeDir opts) config Warp.runSettings settings waiApp diff --git a/package.yaml b/package.yaml index 1238084..974c336 100644 --- a/package.yaml +++ b/package.yaml @@ -29,6 +29,7 @@ ghc-options: dependencies: - base >= 4.7 && < 5 - aeson + - base64-bytestring - json-fleece-aeson - json-fleece-core - blaze-html diff --git a/roux-server.cabal b/roux-server.cabal index 52c78b2..14547c6 100644 --- a/roux-server.cabal +++ b/roux-server.cabal @@ -42,6 +42,7 @@ library build-depends: aeson , base >=4.7 && <5 + , base64-bytestring , blaze-html , blaze-markup , bytestring @@ -79,6 +80,7 @@ executable check-recipes build-depends: aeson , base >=4.7 && <5 + , base64-bytestring , blaze-html , blaze-markup , bytestring @@ -117,6 +119,7 @@ executable roux-server build-depends: aeson , base >=4.7 && <5 + , base64-bytestring , blaze-html , blaze-markup , bytestring @@ -161,6 +164,7 @@ test-suite roux-server-test build-depends: aeson , base >=4.7 && <5 + , base64-bytestring , blaze-html , blaze-markup , bytestring diff --git a/src/Roux/Server.hs b/src/Roux/Server.hs index 6c667d0..1e2827b 100644 --- a/src/Roux/Server.hs +++ b/src/Roux/Server.hs @@ -14,10 +14,11 @@ module Roux.Server ( import Control.Concurrent (forkIO, threadDelay) import Control.Exception (IOException, SomeException, catch, finally, try) import Control.Monad (forM_, forever, when) +import Data.ByteString.Base64 (decode) import Data.ByteString.Builder (lazyByteString) import Data.ByteString.Lazy (ByteString) import qualified Data.ByteString.Lazy as LB -import Data.Char (digitToInt, toLower) +import Data.Char (digitToInt, isAlphaNum, toLower) import Data.IORef (IORef, atomicModifyIORef', newIORef, readIORef, writeIORef) import Data.List (isSuffixOf) import Data.Map.Strict (Map) @@ -29,7 +30,7 @@ import Data.Time.Clock (diffUTCTime, getCurrentTime) import qualified Network.HTTP.Types as HTTP import qualified Network.Wai as Wai -import System.Directory (doesFileExist, findExecutable, makeAbsolute) +import System.Directory (createDirectoryIfMissing, doesFileExist, findExecutable, makeAbsolute, removeFile) import System.FSNotify ( Event (..), EventIsDirectory (..), @@ -45,6 +46,7 @@ import qualified Data.Aeson as A import qualified Data.ByteString as BS import Data.Maybe (mapMaybe) import qualified Data.Text.Encoding as TE +import Roux.Config (AnthropicConfig (..), RouxConfig (..)) import qualified Roux.CooklangPrint as CooklangPrint import qualified Roux.Html as Html import qualified Roux.RecipeIndex as Idx @@ -65,12 +67,12 @@ readRequestBody req = do then pure (reverse acc) else gather (chunk : acc) -{- | Build the WAI 'Application' given a path to the recipe directory. +{- | Build the WAI 'Application' given a path to the recipe directory and config. Recipes are scanned at startup and then watched live via fsnotify. -} -app :: FilePath -> IO Wai.Application -app recipeDir = do +app :: FilePath -> RouxConfig -> IO Wai.Application +app recipeDir config = do putStrLn $ "[roux] scanning recipes from " <> recipeDir recipes <- Idx.scanRecipes recipeDir let count = length recipes @@ -92,7 +94,7 @@ app recipeDir = do forkIO $ watchRecipes absDir state changeLog `catch` \e -> putStrLn $ "[roux] watcher thread crashed: " <> show (e :: SomeException) - pure (router absDir state changeLog) + pure (router absDir config state changeLog) where isRight (Right _) = True isRight (Left _) = False @@ -174,19 +176,21 @@ sseHandler changeLog _request respond = do go `finally` pure () -- | Route requests — SSE endpoint, recipe pages, or import. -router :: FilePath -> IORef (Map FilePath Idx.RecipeInfo) -> ChangeLog -> Wai.Application -router recipeDir state changeLog request respond = +router :: FilePath -> RouxConfig -> IORef (Map FilePath Idx.RecipeInfo) -> ChangeLog -> Wai.Application +router recipeDir config state changeLog request respond = case Wai.pathInfo request of ["events"] -> sseHandler changeLog request respond - ["import"] -> importHandler recipeDir state request respond + ["import"] -> importHandler recipeDir config state request respond _ -> handleRequest state request respond {- | Handle GET and POST requests to /import. GET — show the import form. -POST — scrape the URL, convert to Cooklang, save the file. +POST — dispatch based on input: URL (existing scraper), text, or file (Anthropic). +Files are uploaded as base64-encoded content via hidden form fields +(file-b64, file-name, file-mime) set by client-side JavaScript. -} -importHandler :: FilePath -> IORef (Map FilePath Idx.RecipeInfo) -> Wai.Application -importHandler recipeDir _state request respond = +importHandler :: FilePath -> RouxConfig -> IORef (Map FilePath Idx.RecipeInfo) -> Wai.Application +importHandler recipeDir config _state request respond = case Wai.requestMethod request of "GET" -> respond (htmlResponse (Html.importPage Nothing)) @@ -194,27 +198,47 @@ importHandler recipeDir _state request respond = body <- readRequestBody request let params = parseFormBody body url = maybe "" id (lookup "url" params) - if T.null url - then respond (htmlResponse (Html.importPage (Just ("URL is required" :: Text)))) - else do - putStrLn $ "[roux] import request for " <> T.unpack url - result <- try $ runImportPipeline recipeDir url - case result of - Left (e :: SomeException) -> - respond $ - htmlResponse $ - Html.importResultPage $ - Html.ImportError ("Unexpected error: " <> T.pack (show e)) - Right (Left err) -> - respond $ - htmlResponse $ - Html.importResultPage err - Right (Right filename) -> - respond $ - Wai.responseLBS - HTTP.status303 - [("Location", "/recipes/" <> TE.encodeUtf8 (T.pack filename))] - "" + text = maybe "" id (lookup "text" params) + fileB64 = maybe "" id (lookup "file-b64" params) + fileName = maybe "" id (lookup "file-name" params) + fileMime = maybe "" id (lookup "file-mime" params) + hasFile = not (T.null fileB64) + + putStrLn $ + "[roux] import request: url=" + <> T.unpack url + <> " text=" + <> show (not (T.null text)) + <> " file=" + <> show hasFile + + result <- try $ case (hasFile, not (T.null text), not (T.null url)) of + -- File takes priority, then text, then URL + (True, _, _) -> + runFileImport recipeDir config fileName fileMime fileB64 + (_, True, _) -> + runTextImport recipeDir config text + (_, _, True) -> + runImportPipeline recipeDir url + _ -> + pure (Left (Html.ImportError "Provide a URL, recipe text, or file to import.")) + + case result of + Left (e :: SomeException) -> + respond $ + htmlResponse $ + Html.importResultPage $ + Html.ImportError ("Unexpected error: " <> T.pack (show e)) + Right (Left err) -> + respond $ + htmlResponse $ + Html.importResultPage err + Right (Right filename) -> + respond $ + Wai.responseLBS + HTTP.status303 + [("Location", "/recipes/" <> TE.encodeUtf8 (T.pack filename))] + "" _ -> respond notFound -- | Parse URL-encoded form body into key-value pairs. @@ -336,6 +360,157 @@ runImportPipeline recipeDir url = do putStrLn $ "[roux] successfully imported " <> T.unpack url <> " as " <> filename return $ Right filename +{- | Import a recipe from an uploaded file via the LLM pipeline. +The file content is base64-encoded in the form data. +-} +runFileImport :: + -- | recipe directory + FilePath -> + -- | server config (for Anthropic settings) + RouxConfig -> + -- | original filename + Text -> + -- | MIME type + Text -> + -- | base64-encoded file content + Text -> + IO (Either Html.ImportError FilePath) +runFileImport recipeDir config originalName _mimeType b64Text = do + -- Decode base64 + case decode (TE.encodeUtf8 b64Text) of + Left err -> + pure $ + Left $ + Html.ImportError $ + "Could not decode uploaded file: " <> T.pack err + Right content -> do + -- Ensure recipe dir exists + createDirectoryIfMissing True recipeDir + -- Save the original file + let sourceFilename = map (\c -> if isAlphaNum c || c == '.' || c == '-' || c == '_' then c else '_') (T.unpack originalName) + sourcePath = recipeDir sourceFilename + BS.writeFile sourcePath content + putStrLn $ "[roux] saved uploaded file to " <> sourcePath + -- Call the LLM pipeline with the saved file + case rcAnthropic config of + Nothing -> + pure $ + Left $ + Html.ImportError $ + "LLM import is not configured. Add an anthropic.api_key to the config file." + Just anthropicConfig -> do + result <- runLLMPipeline recipeDir anthropicConfig (Just sourcePath) "" + case result of + Right filename -> pure $ Right filename + Left err -> pure $ Left err + +-- | Import a recipe from pasted text via the LLM pipeline. +runTextImport :: + FilePath -> + RouxConfig -> + Text -> + IO (Either Html.ImportError FilePath) +runTextImport recipeDir config text = do + case rcAnthropic config of + Nothing -> + pure $ + Left $ + Html.ImportError $ + "LLM import is not configured. Add an anthropic.api_key to the config file." + Just anthropicConfig -> + runLLMPipeline recipeDir anthropicConfig Nothing text + +-- | Run the full LLM import pipeline: call convert-to-cooklang -> parse -> save. +runLLMPipeline :: + -- | recipe directory + FilePath -> + -- | API config + AnthropicConfig -> + -- | Just path = uploaded source file to keep + Maybe FilePath -> + -- | recipe text (or empty if file) + Text -> + IO (Either Html.ImportError FilePath) +runLLMPipeline recipeDir anthropicConfig mSourceFile text = do + -- Step 1: locate the script + script <- findConvertScript + -- Step 2: write a temp config for the subprocess + let configPath = recipeDir ".convert-config.yaml" + writeConvertConfig configPath anthropicConfig + -- Step 3: build subprocess args + let args = case mSourceFile of + Just fp -> ["--config", configPath, "--file", fp] + Nothing -> ["--config", configPath, "--text", T.unpack text] + putStrLn $ "[roux] running " <> script <> " for LLM import" + (exitCode, stdoutBytes, stderrBytes) <- + readProcessBytes script args + -- Clean up temp config + _ <- try (removeFile configPath) :: IO (Either IOException ()) + case exitCode of + ExitFailure _ -> do + let errMsg = TE.decodeUtf8 (LB.toStrict stderrBytes) + putStrLn $ "[roux] convert-to-cooklang failed: " <> T.unpack errMsg + return $ + Left $ + Html.ImportError $ + "Could not convert recipe: " <> errMsg + ExitSuccess -> do + -- Step 4: parse the Cooklang output to extract the title + let cooklangText = TE.decodeUtf8 (LB.toStrict stdoutBytes) + case parseRecipeTitle cooklangText of + Nothing -> do + putStrLn "[roux] could not extract title from Cooklang output" + return $ + Left $ + Html.ImportError $ + "Could not determine recipe title from AI output." + Just title -> do + let filename = CooklangPrint.filenameFromTitle title + filepath = recipeDir filename + -- Step 5: write the .cook file + putStrLn $ "[roux] writing imported recipe to " <> filepath + LB.writeFile filepath stdoutBytes + putStrLn $ "[roux] successfully imported recipe as " <> filename + return $ Right filename + +-- | Locate the convert-to-cooklang script. +findConvertScript :: IO FilePath +findConvertScript = do + mPath <- findExecutable "convert-to-cooklang" + case mPath of + Just p -> pure p + Nothing -> pure "scripts/convert-to-cooklang" + +-- | Write a temporary YAML config for the subprocess. +writeConvertConfig :: FilePath -> AnthropicConfig -> IO () +writeConvertConfig path ac = do + let pairs = + [ "anthropic:" + , " api_key: " <> T.unpack (acApiKey ac) + ] + ++ case acBaseUrl ac of + Just url -> [" base_url: " <> T.unpack url] + Nothing -> [] + ++ case acModel ac of + Just m -> [" model: " <> T.unpack m] + Nothing -> [] + writeFile path (unlines pairs) + +{- | Parse the recipe title from Cooklang front matter. +Looks for "title: ..." in the YAML front matter block. +-} +parseRecipeTitle :: Text -> Maybe Text +parseRecipeTitle cooklang = + case T.breakOn "---" cooklang of + (_, afterFirst) -> + let (_separator, inside) = T.breakOn "---" (T.drop 3 afterFirst) + frontMatter = T.take (T.length inside) (T.drop 3 afterFirst) + in case T.breakOn "title:" frontMatter of + (_, afterTitle) -> + let titleLine = T.takeWhile (/= '\n') (T.drop 6 afterTitle) + cleaned = T.strip titleLine + in if T.null cleaned then Nothing else Just cleaned + {- | Watch a directory for .cook file changes and update the shared state. Uses watchDir callback for simplicity. -}