fix: read subprocess output as raw bytes to avoid locale encoding errors
Build and Deploy / build-and-deploy (push) Successful in 1m32s
Build and Deploy / build-and-deploy (push) Successful in 1m32s
readProcessWithExitCode returns stdout/stderr as String, which forces GHC to decode raw bytes using the current locale encoding. In a Docker container with C locale (default), non-ASCII UTF-8 bytes (like 0xE2) cause 'invalid argument' errors. Replace with readProcessBytes which uses createProcess + hSetBinaryMode to capture stdout/stderr as raw lazy ByteStrings, bypassing locale decoding entirely. JSON is decoded directly from raw bytes via A.decode.
This commit is contained in:
+24
-11
@@ -50,7 +50,8 @@ import qualified Roux.Html as Html
|
|||||||
import qualified Roux.RecipeIndex as Idx
|
import qualified Roux.RecipeIndex as Idx
|
||||||
import Roux.SchemaOrg (SchemaOrgRecipe (..), parseSchemaOrgRecipe, schemaOrgToCooklang)
|
import Roux.SchemaOrg (SchemaOrgRecipe (..), parseSchemaOrgRecipe, schemaOrgToCooklang)
|
||||||
import System.Exit (ExitCode (..))
|
import System.Exit (ExitCode (..))
|
||||||
import System.Process (readProcessWithExitCode)
|
import System.IO (hSetBinaryMode)
|
||||||
|
import System.Process (CreateProcess (..), StdStream (CreatePipe), createProcess, proc, waitForProcess)
|
||||||
|
|
||||||
-- | Read the full request body as a lazy ByteString.
|
-- | Read the full request body as a lazy ByteString.
|
||||||
readRequestBody :: Wai.Request -> IO LB.ByteString
|
readRequestBody :: Wai.Request -> IO LB.ByteString
|
||||||
@@ -234,6 +235,22 @@ parseFormBody body =
|
|||||||
in toEnum hexVal : go rest
|
in toEnum hexVal : go rest
|
||||||
go (c : rest) = c : go rest
|
go (c : rest) = c : go rest
|
||||||
|
|
||||||
|
-- | Run a subprocess and capture stdout/stderr as raw bytes (no locale decoding).
|
||||||
|
readProcessBytes :: FilePath -> [String] -> IO (ExitCode, LB.ByteString, LB.ByteString)
|
||||||
|
readProcessBytes cmd args = do
|
||||||
|
let process =
|
||||||
|
(proc cmd args)
|
||||||
|
{ std_out = CreatePipe
|
||||||
|
, std_err = CreatePipe
|
||||||
|
}
|
||||||
|
(Just outH, Just errH, _, ph) <- createProcess process
|
||||||
|
hSetBinaryMode outH True
|
||||||
|
hSetBinaryMode errH True
|
||||||
|
out <- LB.hGetContents outH
|
||||||
|
err <- LB.hGetContents errH
|
||||||
|
ec <- waitForProcess ph
|
||||||
|
return (ec, out, err)
|
||||||
|
|
||||||
-- | Locate the scrape-recipe script: check PATH first, then relative to project.
|
-- | Locate the scrape-recipe script: check PATH first, then relative to project.
|
||||||
findScrapeScript :: IO FilePath
|
findScrapeScript :: IO FilePath
|
||||||
findScrapeScript = do
|
findScrapeScript = do
|
||||||
@@ -250,23 +267,19 @@ runImportPipeline recipeDir url = do
|
|||||||
-- Step 1: locate and run the scraper
|
-- Step 1: locate and run the scraper
|
||||||
script <- findScrapeScript
|
script <- findScrapeScript
|
||||||
putStrLn $ "[roux] importing from " <> T.unpack url <> " using script " <> script
|
putStrLn $ "[roux] importing from " <> T.unpack url <> " using script " <> script
|
||||||
(exitCode, stdout, stderr) <-
|
(exitCode, stdoutBytes, stderrBytes) <-
|
||||||
readProcessWithExitCode
|
readProcessBytes script ["--pretty", T.unpack url]
|
||||||
script
|
|
||||||
["--pretty", T.unpack url]
|
|
||||||
""
|
|
||||||
case exitCode of
|
case exitCode of
|
||||||
ExitFailure _ -> do
|
ExitFailure _ -> do
|
||||||
putStrLn $ "[roux] scrape-recipe exited with code " <> show exitCode <> " for " <> T.unpack url
|
putStrLn $ "[roux] scrape-recipe exited with code " <> show exitCode <> " for " <> T.unpack url
|
||||||
putStrLn $ "[roux] scrape-recipe stderr: " <> stderr
|
putStrLn $ "[roux] scrape-recipe stderr: " <> T.unpack (TE.decodeUtf8 (LB.toStrict stderrBytes))
|
||||||
return $
|
return $
|
||||||
Left $
|
Left $
|
||||||
Html.ImportError $
|
Html.ImportError $
|
||||||
"Could not scrape URL (" <> url <> "). The scraper exited with: " <> T.pack stderr
|
"Could not scrape URL (" <> url <> "). The scraper exited with: " <> TE.decodeUtf8 (LB.toStrict stderrBytes)
|
||||||
ExitSuccess -> do
|
ExitSuccess -> do
|
||||||
-- Step 2: decode stdout (String) as JSON Value
|
-- Step 2: decode stdout (raw bytes) as JSON Value
|
||||||
let jsonBytes = LB.fromStrict (TE.encodeUtf8 (T.pack stdout))
|
let jsonVal = A.decode stdoutBytes :: Maybe A.Value
|
||||||
jsonVal = A.decode jsonBytes :: Maybe A.Value
|
|
||||||
case jsonVal of
|
case jsonVal of
|
||||||
Nothing -> do
|
Nothing -> do
|
||||||
putStrLn $ "[roux] failed to decode JSON from scrape-recipe stdout for " <> T.unpack url
|
putStrLn $ "[roux] failed to decode JSON from scrape-recipe stdout for " <> T.unpack url
|
||||||
|
|||||||
Reference in New Issue
Block a user