I am using PythonNet in a C# application to run multiple Python scripts sequentially, each contained in different modules. However, I am encountering inconsistent MemoryAccessViolation errors, which sometimes occur during program execution, at shutdown, or even seemingly at random times. These errors do not follow a specific pattern and are difficult to reproduce reliably.
When I reuse the same PythonNet instance to run different scripts, errors arise. Attempting to shut down and reload the PythonNet instance also leads to these memory access issues. Additionally, these errors are not caught by try-catch blocks, and the program often crashes with an access violation error code.
Here is the relevant code snippet showing how I initialize and dispose of the Python environment:
/// <summary>
/// Manages the Python environment using pythonnet.
/// </summary>
public class PythonNetEnv : IDisposable
{
/// <summary>
/// Indicates if the environment is initialized.
/// </summary>
public bool IsInitialized { get; private set; }
private readonly PythonEnv _env;
private PythonEngine? _engine;
private string? _originalPath;
private string? _originalPythonhome;
private CancellationTokenSource _cts = new();
/// <summary>
/// Initializes a new instance of the <see cref="PythonNetEnv"/> class.
/// </summary>
/// <param name="pythonEnv">The path to the Python environment directory.</param>
public PythonNetEnv(PythonEnv pythonEnv)
{
_env = pythonEnv;
Initialize();
}
/// <summary>
/// Initializes the Python environment.
/// </summary>
private void Initialize()
{
if (IsInitialized) return;
// Construct the necessary paths
string scriptsPath = Path.Combine(_env.EnvPath, "Scripts");
string libraryPath = Path.Combine(_env.EnvPath, "Library");
string binPath = Path.Combine(_env.EnvPath, "bin");
string executablePath = Path.Combine(_env.EnvPath, "Library", "bin");
string mingwBinPath = Path.Combine(_env.EnvPath, "Library", "mingw-w64", "bin");
// Get the current PATH environment variable
_originalPath = Environment.GetEnvironmentVariable("PATH");
_originalPythonhome = Environment.GetEnvironmentVariable("PYTHONHOME");
// Set the new PATH environment variable
string newPath = $"{_env.EnvPath};{scriptsPath};{libraryPath};{binPath};{executablePath};{mingwBinPath};{_originalPath}";
Environment.SetEnvironmentVariable("PATH", newPath, EnvironmentVariableTarget.Process);
// Set the PYTHONHOME environment variable
Environment.SetEnvironmentVariable("PYTHONHOME", _env.EnvPath, EnvironmentVariableTarget.Process);
// Extract the major and minor version numbers from the provided version string
string[] versionParts = _env.Version.Split('.');
if (versionParts.Length < 2)
throw new ArgumentException("Invalid Python version format. Expected format: main.current.path (e.g., 3.8.20)");
string majorVersion = versionParts[0];
string minorVersion = versionParts[1];
// Construct the Python runtime DLL path based on the version
string pythonDllPath = Path.Combine(_env.EnvPath, $"python{majorVersion}{minorVersion}.dll");
// Explicitly set the Python runtime DLL path
Runtime.PythonDLL = pythonDllPath;
// Set PythonEngine.PythonHome
PythonEngine.PythonHome = _env.EnvPath;
if (!PythonEngine.IsInitialized)
_engine = new PythonEngine();
// Allow Python threads to run independently of the main thread
PythonEngine.BeginAllowThreads();
IsInitialized = true;
}
/// <summary>
/// Runs a Python script from a specific path with specified arguments.
/// </summary>
/// <param name="scriptPath">The path to the Python script.</param>
/// <param name="workingDirectory">The working directory for the script.</param>
/// <param name="arguments">The arguments to pass to the script.</param>
public async Task RunPythonScript(string scriptPath, string workingDirectory, string arguments)
{
if (!IsInitialized) throw new InvalidOperationException("Python environment is not initialized.");
await Task.Run(() =>
{
string currentDictionary = Environment.CurrentDirectory;
using (Py.GIL())
{
try
{
Environment.CurrentDirectory = workingDirectory;
dynamic sys = Py.Import("sys");
dynamic io = Py.Import("io");
string moduleName = Path.GetFileNameWithoutExtension(scriptPath);
Debug.WriteLine($"Importing module: {moduleName}");
dynamic stdout_capture = io.StringIO();
dynamic stderr_capture = io.StringIO();
sys.stdout = stdout_capture;
sys.stderr = stderr_capture;
_cts = new CancellationTokenSource();
_ = Task.Run(() => CaptureOutput(stdout_capture, _cts.Token));
_ = Task.Run(() => CaptureOutput(stderr_capture, _cts.Token));
try
{
ExecutePythonScript(scriptPath);
}
finally
{
_cts.Cancel();
RestoreStandardOutputs(sys, stdout_capture, stderr_capture);
}
}
catch (Exception ex)
{
Console.WriteLine($"Error: {ex.Message}");
Console.WriteLine($"Stack Trace: {ex.StackTrace}");
}
}
Environment.CurrentDirectory = currentDictionary;
});
}
private async Task CaptureOutput(dynamic captureStream, CancellationToken token)
{
string currentOutput = "";
while (!token.IsCancellationRequested)
{
try
{
using (Py.GIL())
{
var currentOutputValue = captureStream.getvalue();
if (currentOutputValue != null)
currentOutput = currentOutputValue.ToString(); // It happens somethimes here, while the code is running. Throws an error but the try catch does also not prevent exiting of the program.
}
}
catch (System.AccessViolationException ex) { Debug.WriteLine(ex); }
// Some more unrelated code...
}
}
private static void ExecutePythonScript(string scriptPath)
{
using (Py.GIL())
{
using dynamic scope = Py.CreateScope();
scope.Set("__name__", "__main__");
scope.Exec(File.ReadAllText(scriptPath)); // It happens somethimes here, while the code is running. Throws an error but the try catch does also not prevent exiting of the program.
}
}
private static void RestoreStandardOutputs(dynamic sys, dynamic stdout_capture, dynamic stderr_capture)
{
using (Py.GIL())
{
sys.stdout = sys.__stdout__;
sys.stderr = sys.__stderr__;
_ = stdout_capture.getvalue().ToString();
_ = stderr_capture.getvalue().ToString();
}
}
/// <summary>
/// Disposes of the Python environment, shutting down the Python engine.
/// </summary>
public void Dispose()
{
if (!IsInitialized)
return;
try
{
_cts.Dispose();
Debug.WriteLine("Shutdown");
PythonEngine.Shutdown(); // Somewhere here does it happen at most. The catch does not work.
_engine?.Dispose();
}
catch (Exception ex)
{
Debug.WriteLine(ex);
}
finally
{
Environment.SetEnvironmentVariable("PATH", _originalPath, EnvironmentVariableTarget.Process);
Environment.SetEnvironmentVariable("PYTHONHOME", _originalPythonhome, EnvironmentVariableTarget.Process);
}
}
}
Additional Details:
- The errors do not happen on the first or second call but usually occur on the third call.
- I create a PythonNet object, let it run, dispose it, and start a new PythonNet object.
- I have marked the lines in the code where the errors happen, but I don't understand why it works sometimes 4 times and sometimes only once. This is completely random.
- Sometimes, the program ends with an access violation error code without throwing any errors.
- The program runs to the end and then shuts down, where it most often exits with access violations.
Questions:
- How can I properly reset or restart PythonNet between different Python script executions to avoid these memory access errors?
- Is there a way to ensure that each Python script runs in a completely isolated environment within PythonNet?
- What might be causing these random access violations, and how can I prevent them?
I am grateful for any suggestions or insights into resolving this issue.
