Declutterfier! Saves Data!

--- a PPN by Garber Painting Akron. With Image Size Reduction included!

URL: http://github.com/pythonnet/pythonnet/pull/407.patch

int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime.UCS; + + IntPtr mem = Marshal.AllocHGlobal(memSize); + try + { + // Preparing array of pointers to strings + IntPtr curStrPtr = mem + argv.Length * IntPtr.Size; + for (var i = 0; i < argv.Length; i++) + { + Encoding encoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + byte[] bStr = encoding.GetBytes(argv[i] + "\0"); + Marshal.Copy(bStr, 0, curStrPtr, bStr.Length); + Marshal.WriteIntPtr(mem + i * IntPtr.Size, curStrPtr); + curStrPtr += bStr.Length; + } + } + catch (Exception) + { + Marshal.FreeHGlobal(mem); + throw; + } + + return mem; + } + + public static ICustomMarshaler GetInstance(string cookie) + { + return Instance; + } + } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 50e09dd95..3d5eda0fe 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -667,41 +667,8 @@ internal unsafe static extern IntPtr public unsafe static extern int Py_Main( int argc, - IntPtr lplpargv + [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv ); - - public static int Py_Main(int argc, string[] argv) - { - // Totally ignoring argc. - argc = argv.Length; - - var allStringsLength = 0; - foreach (string x in argv) - { - allStringsLength += x.Length + 1; - } - int requiredSize = IntPtr.Size * argc + allStringsLength * UCS; - IntPtr mem = Marshal.AllocHGlobal(requiredSize); - try - { - // Preparing array of pointers to UTF32 strings. - IntPtr curStrPtr = mem + argc * IntPtr.Size; - for (var i = 0; i < argv.Length; i++) - { - // Unicode or UTF8 work - Encoding enc = UCS == 2 ? Encoding.Unicode : Encoding.UTF32; - byte[] zstr = enc.GetBytes(argv[i] + "\0"); - Marshal.Copy(zstr, 0, curStrPtr, zstr.Length); - Marshal.WriteIntPtr(mem + IntPtr.Size * i, curStrPtr); - curStrPtr += zstr.Length; - } - return Py_Main(argc, mem); - } - finally - { - Marshal.FreeHGlobal(mem); - } - } #elif PYTHON2 [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, CharSet = CharSet.Ansi)] @@ -2078,41 +2045,9 @@ internal unsafe static extern IntPtr internal unsafe static extern void PySys_SetArgvEx( int argc, - IntPtr lplpargv, + [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(StrArrayMarshaler))] string[] argv, int updatepath ); - - internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) - { - // Totally ignoring argc. - argc = argv.Length; - - var allStringsLength = 0; - foreach (string x in argv) - { - allStringsLength += x.Length + 1; - } - int requiredSize = IntPtr.Size * argc + allStringsLength * UCS; - IntPtr mem = Marshal.AllocHGlobal(requiredSize); - try - { - // Preparing array of pointers to UTF32 strings. - IntPtr curStrPtr = mem + argc * IntPtr.Size; - for (var i = 0; i < argv.Length; i++) - { - Encoding enc = UCS == 2 ? Encoding.Unicode : Encoding.UTF32; - byte[] zstr = enc.GetBytes(argv[i] + "\0"); - Marshal.Copy(zstr, 0, curStrPtr, zstr.Length); - Marshal.WriteIntPtr(mem + IntPtr.Size * i, curStrPtr); - curStrPtr += zstr.Length; - } - PySys_SetArgvEx(argc, mem, updatepath); - } - finally - { - Marshal.FreeHGlobal(mem); - } - } #elif PYTHON2 [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, CharSet = CharSet.Ansi)] From 74440a6b90beaaa32a2df8bc6b3660ce5edafbcb Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Sat, 25 Feb 2017 10:05:16 -0700 Subject: [PATCH 3/8] Refactor Marshals --- src/runtime/CustomMarshaler.cs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index f776cb1e3..1004122cd 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Runtime.InteropServices; using System.Text; @@ -41,10 +42,10 @@ public int GetNativeDataSize() public class StrMarshaler : MarshalerBase { private static readonly MarshalerBase Instance = new StrMarshaler(); + private static readonly Encoding PyEncoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32; public override IntPtr MarshalManagedToNative(object managedObj) { - Encoding encoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32; var s = managedObj as string; if (s == null) @@ -52,12 +53,7 @@ public override IntPtr MarshalManagedToNative(object managedObj) return IntPtr.Zero; } - int minByteCount = encoding.GetMaxByteCount(1); - char[] cStr = s.ToCharArray(0, s.Length); - byte[] bStr = new byte[encoding.GetByteCount(cStr) + minByteCount]; - encoding.GetBytes(cStr, 0, cStr.Length, bStr, 0); - DebugUtil.PrintHexBytes(bStr); - + byte[] bStr = PyEncoding.GetBytes(s + "\0"); IntPtr mem = Marshal.AllocHGlobal(bStr.Length); try { @@ -86,6 +82,7 @@ public static ICustomMarshaler GetInstance(string cookie) public class StrArrayMarshaler : MarshalerBase { private static readonly MarshalerBase Instance = new StrArrayMarshaler(); + private static readonly Encoding PyEncoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32; public override IntPtr MarshalManagedToNative(object managedObj) { @@ -96,11 +93,7 @@ public override IntPtr MarshalManagedToNative(object managedObj) return IntPtr.Zero; } - var totalStrLength = 0; - foreach (string arg in argv) - { - totalStrLength += arg.Length + 1; - } + int totalStrLength = argv.Sum(arg => arg.Length + 1); int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime.UCS; IntPtr mem = Marshal.AllocHGlobal(memSize); @@ -110,8 +103,7 @@ public override IntPtr MarshalManagedToNative(object managedObj) IntPtr curStrPtr = mem + argv.Length * IntPtr.Size; for (var i = 0; i < argv.Length; i++) { - Encoding encoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32; - byte[] bStr = encoding.GetBytes(argv[i] + "\0"); + byte[] bStr = PyEncoding.GetBytes(argv[i] + "\0"); Marshal.Copy(bStr, 0, curStrPtr, bStr.Length); Marshal.WriteIntPtr(mem + i * IntPtr.Size, curStrPtr); curStrPtr += bStr.Length; From e487076d90ac86f15150c1b41113565364fb7850 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Sat, 25 Feb 2017 12:58:27 -0700 Subject: [PATCH 4/8] Add ICustomMarshaler Utf8Marshaler Refactor PyString_FromStringAndSize Link explains why `MarshalAs(UnmanagedType.LPWStr)` or `CharSet.Unicode` don't work http://stackoverflow.com/a/25128147/5208670 --- src/runtime/CustomMarshaler.cs | 45 ++++++++++++++++++++++++++++++++++ src/runtime/runtime.cs | 23 +++++------------ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index 1004122cd..a4947a909 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -123,4 +123,49 @@ public static ICustomMarshaler GetInstance(string cookie) return Instance; } } + + + //github.com/

+ //github.com/ Custom Marshaler to deal with Managed String to Native + //github.com/ conversion on UTF-8. Use on functions that expect UTF-8 encoded + //github.com/ strings like `PyUnicode_FromStringAndSize` + //github.com/

+ //github.com/ + //github.com/ If instead we used `MarshalAs(UnmanagedType.LPWStr)` the output to + //github.com/ `foo` would be `f\x00o\x00o\x00`. + //github.com/ + public class Utf8Marshaler : MarshalerBase + { + private static readonly MarshalerBase Instance = new Utf8Marshaler(); + private static readonly Encoding PyEncoding = Encoding.UTF8; + + public override IntPtr MarshalManagedToNative(object managedObj) + { + var s = managedObj as string; + + if (s == null) + { + return IntPtr.Zero; + } + + byte[] bStr = PyEncoding.GetBytes(s + "\0"); + IntPtr mem = Marshal.AllocHGlobal(bStr.Length); + try + { + Marshal.Copy(bStr, 0, mem, bStr.Length); + } + catch (Exception) + { + Marshal.FreeHGlobal(mem); + throw; + } + + return mem; + } + + public static ICustomMarshaler GetInstance(string cookie) + { + return Instance; + } + } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 3d5eda0fe..50c31ddb4 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1508,23 +1508,12 @@ internal static IntPtr PyBytes_AS_STRING(IntPtr ob) return ob + BytesOffset.ob_sval; } - internal static IntPtr PyString_FromStringAndSize(string value, int length) - { - // copy the string into an unmanaged UTF-8 buffer - int len = Encoding.UTF8.GetByteCount(value); - byte[] buffer = new byte[len + 1]; - Encoding.UTF8.GetBytes(value, 0, value.Length, buffer, 0); - IntPtr nativeUtf8 = Marshal.AllocHGlobal(buffer.Length); - try - { - Marshal.Copy(buffer, 0, nativeUtf8, buffer.Length); - return PyUnicode_FromStringAndSize(nativeUtf8, length); - } - finally - { - Marshal.FreeHGlobal(nativeUtf8); - } - } + [DllImport(Runtime.dll, EntryPoint = "PyUnicode_FromStringAndSize")] + internal static extern IntPtr + PyString_FromStringAndSize( + [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value, + int size + ); [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, CharSet = CharSet.Unicode)] From ccd4521298e87c53d1217670acbc3963371020a3 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Sat, 25 Feb 2017 19:57:52 -0700 Subject: [PATCH 5/8] Match PyUnicode_AsUnicode signature in UCS2/UCS4 --- src/runtime/runtime.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 50c31ddb4..13549b581 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1572,7 +1572,7 @@ internal unsafe static extern int [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, CharSet = CharSet.Unicode)] - internal unsafe static extern char* + internal unsafe static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, @@ -1613,7 +1613,7 @@ internal unsafe static extern int [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicodeUCS2_AsUnicode", ExactSpelling = true)] - internal unsafe static extern char* + internal unsafe static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, @@ -1749,9 +1749,9 @@ internal unsafe static string GetManagedString(IntPtr op) Marshal.Copy(p, buffer, 0, size); return Encoding.UTF32.GetString(buffer, 0, size); #elif UCS2 - char* p = Runtime.PyUnicode_AsUnicode(op); - int size = Runtime.PyUnicode_GetSize(op); - return new String(p, 0, size); + IntPtr p = Runtime.PyUnicode_AsUnicode(op); + int length = Runtime.PyUnicode_GetSize(op); + return Marshal.PtrToStringUni(p, length); #endif } From 07f87de2ae0456d7e5c09333a09ab6bbbc11cf1f Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Sat, 25 Feb 2017 23:09:48 -0700 Subject: [PATCH 6/8] Refactor GetManagedString --- src/runtime/runtime.cs | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 13549b581..a9dcdd81e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1725,7 +1725,20 @@ internal static IntPtr PyUnicode_FromString(string s) return PyUnicode_FromUnicode(s, (s.Length)); } - internal unsafe static string GetManagedString(IntPtr op) + //github.com/

+ //github.com/ Function to access the internal PyUnicode/PyString object and + //github.com/ convert it to a managed string with the correct encoding. + //github.com/

+ //github.com/ + //github.com/ We can't easily do this through through the CustomMarshaler's on + //github.com/ the returns because will have access to the IntPtr but not size. + //github.com/ + //github.com/ For PyUnicodeType, we can't convert with Marshal.PtrToStringUni + //github.com/ since it only works for UCS2. + //github.com/ + //github.com/ PyStringType or PyUnicodeType object to convert + //github.com/ Managed String + internal static string GetManagedString(IntPtr op) { IntPtr type = PyObject_TYPE(op); @@ -1741,18 +1754,15 @@ internal unsafe static string GetManagedString(IntPtr op) if (type == Runtime.PyUnicodeType) { -#if UCS4 - IntPtr p = Runtime.PyUnicode_AsUnicode(op); - int length = Runtime.PyUnicode_GetSize(op); - int size = length * 4; - byte[] buffer = new byte[size]; + Encoding encoding = UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + + IntPtr p = PyUnicode_AsUnicode(op); + int length = PyUnicode_GetSize(op); + + int size = length * UCS; + var buffer = new byte[size]; Marshal.Copy(p, buffer, 0, size); - return Encoding.UTF32.GetString(buffer, 0, size); -#elif UCS2 - IntPtr p = Runtime.PyUnicode_AsUnicode(op); - int length = Runtime.PyUnicode_GetSize(op); - return Marshal.PtrToStringUni(p, length); -#endif + return encoding.GetString(buffer, 0, size); } return null; From edafdf2db546908e4cc60f55656a0125e25bac4f Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Sun, 26 Feb 2017 00:18:37 -0700 Subject: [PATCH 7/8] Remove internal PyUnicode_AS_UNICODE Its redundant with PyUnicode_AsUnicode now that the signature is fixed between UCS2/UCS4. Apply char conversion that work on both UCS2/UCS4 --- src/runtime/converter.cs | 19 ++++--------------- src/runtime/runtime.cs | 24 ------------------------ 2 files changed, 4 insertions(+), 39 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index aeaf2d871..8b58e5e9a 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -599,21 +599,10 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo { if (Runtime.PyUnicode_GetSize(value) == 1) { - op = Runtime.PyUnicode_AS_UNICODE(value); - if (Runtime.UCS == 2) // Don't trust linter, statement not always true. - { - // 2011-01-02: Marshal as character array because the cast - // result = (char)Marshal.ReadInt16(op); throws an OverflowException - // on negative numbers with Check Overflow option set on the project - Char[] buff = new Char[1]; - Marshal.Copy(op, buff, 0, 1); - result = buff[0]; - } - else // UCS4 - { - // XXX this is probably NOT correct? - result = (char)Marshal.ReadInt32(op); - } + op = Runtime.PyUnicode_AsUnicode(value); + Char[] buff = new Char[1]; + Marshal.Copy(op, buff, 0, 1); + result = buff[0]; return true; } goto type_error; diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index a9dcdd81e..9558a5ac3 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1575,12 +1575,6 @@ internal unsafe static extern int internal unsafe static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); - [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyUnicode_AsUnicode", - ExactSpelling = true, CharSet = CharSet.Unicode)] - internal unsafe static extern IntPtr - PyUnicode_AS_UNICODE(IntPtr op); - [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, CharSet = CharSet.Unicode)] internal unsafe static extern IntPtr @@ -1616,12 +1610,6 @@ internal unsafe static extern int internal unsafe static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); - [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyUnicodeUCS2_AsUnicode", - ExactSpelling = true, CharSet = CharSet.Unicode)] - internal unsafe static extern IntPtr - PyUnicode_AS_UNICODE(IntPtr op); - [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicodeUCS2_FromOrdinal", ExactSpelling = true, CharSet = CharSet.Unicode)] @@ -1663,12 +1651,6 @@ internal unsafe static extern int internal unsafe static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); - [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyUnicode_AsUnicode", - ExactSpelling = true, CharSet = CharSet.Unicode)] - internal unsafe static extern IntPtr - PyUnicode_AS_UNICODE(IntPtr op); - [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, ExactSpelling = true, CharSet = CharSet.Unicode)] internal unsafe static extern IntPtr @@ -1707,12 +1689,6 @@ internal unsafe static extern int internal unsafe static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); - [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyUnicodeUCS4_AsUnicode", - ExactSpelling = true, CharSet = CharSet.Unicode)] - internal unsafe static extern IntPtr - PyUnicode_AS_UNICODE(IntPtr op); - [DllImport(Runtime.dll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicodeUCS4_FromOrdinal", ExactSpelling = true, CharSet = CharSet.Unicode)] From 78939c599eecdeccad0384feb4f584fd36171024 Mon Sep 17 00:00:00 2001 From: Victor Uriarte Date: Sun, 26 Feb 2017 10:30:06 -0700 Subject: [PATCH 8/8] Refactor Encoding check This won't change during runtime. --- src/runtime/CustomMarshaler.cs | 4 ++-- src/runtime/runtime.cs | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index a4947a909..475fa05eb 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -42,7 +42,7 @@ public int GetNativeDataSize() public class StrMarshaler : MarshalerBase { private static readonly MarshalerBase Instance = new StrMarshaler(); - private static readonly Encoding PyEncoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + private static readonly Encoding PyEncoding = Runtime.PyEncoding; public override IntPtr MarshalManagedToNative(object managedObj) { @@ -82,7 +82,7 @@ public static ICustomMarshaler GetInstance(string cookie) public class StrArrayMarshaler : MarshalerBase { private static readonly MarshalerBase Instance = new StrArrayMarshaler(); - private static readonly Encoding PyEncoding = Runtime.UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + private static readonly Encoding PyEncoding = Runtime.PyEncoding; public override IntPtr MarshalManagedToNative(object managedObj) { diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 9558a5ac3..337b82fbf 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -170,6 +170,11 @@ public class Runtime internal static bool IsPython2; internal static bool IsPython3; + //github.com/

+ //github.com/ Encoding to use to convert Unicode to/from Managed to Native + //github.com/

+ internal static readonly Encoding PyEncoding = UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + //github.com/

//github.com/ Initialize the runtime... //github.com/

@@ -1730,15 +1735,13 @@ internal static string GetManagedString(IntPtr op) if (type == Runtime.PyUnicodeType) { - Encoding encoding = UCS == 2 ? Encoding.Unicode : Encoding.UTF32; - IntPtr p = PyUnicode_AsUnicode(op); int length = PyUnicode_GetSize(op); int size = length * UCS; var buffer = new byte[size]; Marshal.Copy(p, buffer, 0, size); - return encoding.GetString(buffer, 0, size); + return PyEncoding.GetString(buffer, 0, size); } return null; pFad - Phonifier reborn

Pfad - The Proxy pFad © 2024 Your Company Name. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.

Alternative Proxies: