From e2749229e4c7de052115af8c9a408341eb4f2ce6 Mon Sep 17 00:00:00 2001 From: Alexey Khudyakov Date: Sat, 4 Jan 2025 22:48:12 +0300 Subject: [PATCH 1/3] Embed python script into source instead Nix does something _evil_ to extra-data-files. So let try to work around it. Besides inline-r takes similar approach --- inline-python.cabal | 6 ++---- py/bound-vars.py | 11 ++++++++--- src/Python/Internal/EvalQQ.hs | 22 +++++++++++++++++++--- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/inline-python.cabal b/inline-python.cabal index 71d86ac..7b384a0 100644 --- a/inline-python.cabal +++ b/inline-python.cabal @@ -21,7 +21,6 @@ extra-doc-files: ChangeLog.md extra-source-files: include/inline-python.h -data-files: py/bound-vars.py source-repository head @@ -55,6 +54,8 @@ Library , transformers >=0.4 , inline-c >=0.9.1 , template-haskell -any + , text >=2 + , bytestring hs-source-dirs: src include-dirs: include c-sources: cbits/python.c @@ -72,9 +73,6 @@ Library Python.Internal.Program Python.Internal.Types Python.Internal.Util - Paths_inline_python - Autogen-modules: - Paths_inline_python ---------------------------------------------------------------- library test diff --git a/py/bound-vars.py b/py/bound-vars.py index 56e5814..311810f 100644 --- a/py/bound-vars.py +++ b/py/bound-vars.py @@ -4,15 +4,20 @@ import ast import sys import re +import base64 mode = sys.argv[1] is_hs = re.compile('.*_hs$') -code = ast.parse(sys.stdin.read(), '', mode) def extract_hs_vars(code): for node in ast.walk(code): if isinstance(node, ast.Name) and is_hs.match(node.id): yield node.id -for nm in set(extract_hs_vars(code)): - print(nm) +def print_hs_vars(src): + code = ast.parse(src, '', mode) + for nm in set(extract_hs_vars(code)): + print(nm) + +def decode_and_print(codeB64): + print_hs_vars(base64.b16decode(codeB64, casefold=True).decode('utf8')) diff --git a/src/Python/Internal/EvalQQ.hs b/src/Python/Internal/EvalQQ.hs index 0087119..4b49fdc 100644 --- a/src/Python/Internal/EvalQQ.hs +++ b/src/Python/Internal/EvalQQ.hs @@ -17,7 +17,11 @@ module Python.Internal.EvalQQ import Control.Monad.IO.Class import Control.Monad.Trans.Class import Control.Monad.Trans.Cont +import Data.Bits import Data.Char +import Data.ByteString qualified as BS +import Data.Text qualified as T +import Data.Text.Encoding qualified as T import Foreign.C.Types import Foreign.Ptr import Foreign.Storable @@ -34,7 +38,7 @@ import Python.Internal.Types import Python.Internal.Program import Python.Internal.Eval import Python.Inline.Literal -import Paths_inline_python (getDataFileName) + ---------------------------------------------------------------- C.context (C.baseCtx <> pyCtx) @@ -148,6 +152,11 @@ basicDecref o = Py [CU.exp| void { Py_DECREF($(PyObject* o)) } |] -- TH generator ---------------------------------------------------------------- +script :: String +script = $( do let path = "py/bound-vars.py" + TH.addDependentFile path + TH.lift =<< TH.runIO (readFile path) + ) -- | Generate TH splice which updates python environment dictionary -- and returns python source code. @@ -155,9 +164,16 @@ expQQ :: String -- ^ Python evaluation mode: @exec@/@eval@ -> String -- ^ Python source code -> TH.Q TH.Exp expQQ mode src = do - script <- liftIO $ getDataFileName "py/bound-vars.py" antis <- liftIO $ do - (code, stdout, stderr) <- readProcessWithExitCode "python" [script, mode] src + (code, stdout, stderr) <- readProcessWithExitCode "python" ["-", mode] + $ unlines [ script + , "decode_and_print('" <> + concat [ [ intToDigit $ fromIntegral (w `shiftR` 4) + , intToDigit $ fromIntegral (w .&. 15) ] + | w <- BS.unpack $ T.encodeUtf8 $ T.pack src + ] + <> "')" + ] case code of ExitSuccess -> pure $ words stdout ExitFailure{} -> error stderr From eec21cff9b282991bb2e0957f90eca84857c7de8 Mon Sep 17 00:00:00 2001 From: Alexey Khudyakov Date: Sat, 4 Jan 2025 23:08:07 +0300 Subject: [PATCH 2/3] Explain witchery --- src/Python/Internal/EvalQQ.hs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Python/Internal/EvalQQ.hs b/src/Python/Internal/EvalQQ.hs index 4b49fdc..dff2988 100644 --- a/src/Python/Internal/EvalQQ.hs +++ b/src/Python/Internal/EvalQQ.hs @@ -165,6 +165,10 @@ expQQ :: String -- ^ Python evaluation mode: @exec@/@eval@ -> TH.Q TH.Exp expQQ mode src = do antis <- liftIO $ do + -- We've embedded script into library and we need to pass source + -- code of QQ to a script. It can contain whatever symbols so to + -- be safe it's base16 encode. This encoding is very simple and we + -- don't care much about efficiency here (code, stdout, stderr) <- readProcessWithExitCode "python" ["-", mode] $ unlines [ script , "decode_and_print('" <> From 6baccaa5aad37a277bbcac27e499eaafe748b05c Mon Sep 17 00:00:00 2001 From: Alexey Khudyakov Date: Sat, 4 Jan 2025 23:12:08 +0300 Subject: [PATCH 3/3] Check that intpreter is initialized before runnig python code --- src/Python/Internal/Eval.hs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Python/Internal/Eval.hs b/src/Python/Internal/Eval.hs index 12a394a..65b82be 100644 --- a/src/Python/Internal/Eval.hs +++ b/src/Python/Internal/Eval.hs @@ -123,17 +123,24 @@ C.include "" runPy :: Py a -> IO a -- See NOTE: [Python and threading] runPy py - -- Multithreaded RTS - | rtsSupportsBoundThreads = runInBoundThread $ mask_ $ unPy $ ensureGIL py - -- Single-threaded RTS - | otherwise = mask_ $ unPy $ ensureGIL py - + | rtsSupportsBoundThreads = runInBoundThread go -- Multithreaded RTS + | otherwise = go -- Single-threaded RTS + where + -- We check whether interpreter is initialized. Throw exception if + -- it wasn't. Better than segfault isn't it? + go = mask_ $ checkInitialized >> unPy (ensureGIL py) -- | Execute python action. This function is unsafe and should be only -- called in thread of interpreter. unPy :: Py a -> IO a unPy (Py io) = io +checkInitialized :: IO () +checkInitialized = + [CU.exp| int { Py_IsInitialized() } |] >>= \case + 0 -> error "Python is not initialized" + _ -> pure () + ----------------------------------------------------------------