V0.3 temp

.github/workflows/build_whl.yml

@@ -9,8 +9,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pypy3.7", "pypy3.8"]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.7", "pypy3.8", "pypy3.9", "pypy3.10"]
+        os: [ubuntu-22.04, macos-12, macos-14, windows-2019]
 
     steps:
     - uses: actions/checkout@v3
@@ -27,11 +27,6 @@ jobs:
         python -m pip install --upgrade wheel
         pip install flake8 pytest
         pip install cython
-    - name: check unistd
-      if: ${{ runner.os == 'Windows' }}
-      run: |
-        echo "prepare unistd.h"
-        python prepare_unistd.py
     - name: build_whl
       run: |
         python setup.py sdist bdist_wheel --use-cython --use-cffi

.github/workflows/unitest.yml

@@ -9,8 +9,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "pypy3.7", "pypy3.8"]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.7", "pypy3.8", "pypy3.9", "pypy3.10"]
+        os: [ubuntu-22.04, macos-12, macos-14, windows-2019]
       fail-fast: false
 
     steps:
@@ -28,11 +28,6 @@ jobs:
         python -m pip install --upgrade wheel
         pip install flake8 pytest
         pip install cython
-    - name: check unistd
-      if: ${{ runner.os == 'Windows' }}
-      run: |
-          echo "prepare unistd.h"
-          python prepare_unistd.py
     - name: build_whl
       run: |
         python setup.py install --use-cython --use-cffi

.github/workflows/upload.yml

@@ -9,8 +9,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.7", "pypy3.8", "pypy3.9", "pypy3.10"]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.7", "pypy3.8", "pypy3.9", "pypy3.10"]
+        os: [ubuntu-22.04, macos-12, macos-14, windows-2019]
       fail-fast: false
 
     steps:
@@ -29,11 +29,6 @@ jobs:
         pip install flake8 pytest
         pip install twine
         pip install cython
-    - name: check unistd
-      if: ${{ runner.os == 'Windows' }}
-      run: |
-        echo "prepare unistd.h"
-        python prepare_unistd.py
     - name: build_whl
       run: |
         python setup.py sdist bdist_wheel --use-cython --use-cffi

.gitignore

@@ -14,4 +14,5 @@ lib/
 *.dll
 *.lib
 *.a
-*.so
+*.so
+*.egg-info

changename.py

@@ -1,8 +1,20 @@
 import os
+import platform
 
 for f in os.listdir("dist"):
     if "linux" in f:
         os.rename(
             os.path.join("dist", f),
             os.path.join("dist", f.replace("linux", "manylinux2014")),
         )
+    elif "macosx" in f:
+        if platform.machine() == "x86_64":
+            os.rename(
+                os.path.join("dist", f),
+                os.path.join("dist", f.replace("universal2", "x86_64")),
+            )
+        else:
+            os.rename(
+                os.path.join("dist", f),
+                os.path.join("dist", f.replace("universal2", "arm64")),
+            )

pybase16384/__init__.py

@@ -1,54 +1,24 @@
-import os
-import platform
 from io import BytesIO
 
-impl = platform.python_implementation()
-
-
-def _should_use_cffi() -> bool:
-    ev = os.getenv("B14_USE_CFFI")
-    if ev is not None:
-        return True
-    if impl == "CPython":
-        return False
-    else:
-        return True
-
-
-if not _should_use_cffi():
-    from pybase16384.backends.cython import (
-        _decode,
-        _decode_into,
-        _encode,
-        _encode_into,
-        decode_fd,
-        decode_file,
-        decode_len,
-        decode_local_file,
-        encode_fd,
-        encode_file,
-        encode_len,
-        encode_local_file,
-        is_64bits,
-    )
-else:
-    from pybase16384.backends.cffi import (
-        _decode,
-        _decode_into,
-        _encode,
-        _encode_into,
-        decode_fd,
-        decode_file,
-        decode_len,
-        decode_local_file,
-        encode_fd,
-        encode_file,
-        encode_len,
-        encode_local_file,
-        is_64bits,
-    )
-
-__version__ = "0.3.3"
+from pybase16384.backends import (
+    _decode,
+    _decode_into,
+    _decode_parallel,
+    _encode,
+    _encode_into,
+    _encode_parallel,
+    decode_fd,
+    decode_file,
+    decode_len,
+    decode_local_file,
+    encode_fd,
+    encode_file,
+    encode_len,
+    encode_local_file,
+    is_64bits,
+)
+
+__version__ = "0.3.4"
 
 
 def encode(data: bytes) -> bytes:

pybase16384/backends/__init__.py

@@ -1 +1,52 @@
-# Make cython happy, otherwise couldn't find pxd files
+import os
+import platform
+
+impl = platform.python_implementation()
+
+
+def _should_use_cffi() -> bool:
+    ev = os.getenv("B14_USE_CFFI")
+    if ev is not None:
+        return True
+    if impl == "CPython":
+        return False
+    else:
+        return True
+
+
+if not _should_use_cffi():
+    from pybase16384.backends.cython import (
+        _decode,
+        _decode_into,
+        _decode_parallel,
+        _encode,
+        _encode_into,
+        _encode_parallel,
+        decode_fd,
+        decode_file,
+        decode_len,
+        decode_local_file,
+        encode_fd,
+        encode_file,
+        encode_len,
+        encode_local_file,
+        is_64bits,
+    )
+else:
+    from pybase16384.backends.cffi import (
+        _decode,
+        _decode_into,
+        _decode_parallel,
+        _encode,
+        _encode_into,
+        _encode_parallel,
+        decode_fd,
+        decode_file,
+        decode_len,
+        decode_local_file,
+        encode_fd,
+        encode_file,
+        encode_len,
+        encode_local_file,
+        is_64bits,
+    )

pybase16384/backends/cffi/__init__.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import IO
 
-from pybase16384.backends.cffi._core_cffi import ffi, lib
+from pybase16384.backends.cffi._core import ffi, lib
 
 __version__ = "0.1.0"
 
@@ -19,14 +19,12 @@ def _encode(data: bytes) -> bytes:
     output_buf = ffi.new(f"char[{output_size}]")
     if output_buf == ffi.NULL:
         raise MemoryError
-    count = lib.base16384_encode(ffi.from_buffer(data), length, output_buf, output_size)
+    count = lib.base16384_encode(ffi.from_buffer(data), length, output_buf)
     return ffi.unpack(output_buf, count)
 
 
 def _encode_into(data: bytes, out: bytearray) -> int:
-    return lib.base16384_encode(
-        ffi.from_buffer(data), len(data), ffi.from_buffer(out), len(out)
-    )
+    return lib.base16384_encode(ffi.from_buffer(data), len(data), ffi.from_buffer(out))
 
 
 def _decode(data: bytes) -> bytes:
@@ -35,14 +33,12 @@ def _decode(data: bytes) -> bytes:
     output_buf = ffi.new(f"char[{output_size}]")
     if output_buf == ffi.NULL:
         raise MemoryError
-    count = lib.base16384_decode(ffi.from_buffer(data), length, output_buf, output_size)
+    count = lib.base16384_decode(ffi.from_buffer(data), length, output_buf)
     return ffi.unpack(output_buf, count)
 
 
 def _decode_into(data: bytes, out: bytearray) -> int:
-    return lib.base16384_decode(
-        ffi.from_buffer(data), len(data), ffi.from_buffer(out), len(out)
-    )
+    return lib.base16384_decode(ffi.from_buffer(data), len(data), ffi.from_buffer(out))
 
 
 def is_64bits() -> bool:
@@ -92,9 +88,7 @@ def encode_file(input: IO, output: IO, write_head: bool = False, buf_rate: int =
                 input.seek(-size, 1)
                 continue
 
-        count = lib.base16384_encode(
-            ffi.from_buffer(chunk), size, output_buf, output_size
-        )
+        count = lib.base16384_encode(ffi.from_buffer(chunk), size, output_buf)
         output.write(ffi.unpack(output_buf, count))
         if size < 7:
             break
@@ -146,9 +140,7 @@ def decode_file(input: IO, output: IO, buf_rate: int = 10):
             else:
                 input.seek(-2, 1)
 
-        count = lib.base16384_decode(
-            ffi.from_buffer(chunk), size, output_buf, output_size
-        )
+        count = lib.base16384_decode(ffi.from_buffer(chunk), size, output_buf)
         output.write(ffi.unpack(output_buf, count))
 
 
@@ -216,3 +208,11 @@ def decode_fd(inp: int, out: int) -> None:
     ret = lib.base16384_decode_fd(inp, out, encbuf, decbuf)
     if ret != lib.base16384_err_ok:
         raise ValueError(err_to_str(ret))
+
+
+def _decode_parallel(data: bytes, num_threads: int = 2) -> bytes:
+    pass
+
+
+def _encode_parallel(data: bytes, num_threads: int = 2) -> bytes:
+    pass

pybase16384/backends/cffi/build.py

@@ -46,8 +46,8 @@
 typedef enum base16384_err_t base16384_err_t;
 int base16384_encode_len(int dlen);
 int base16384_decode_len(int dlen, int offset);
-int base16384_encode(const char* data, int dlen, char* buf, int blen);
-int base16384_decode(const char* data, int dlen, char* buf, int blen);
+int base16384_encode(const char* data, int dlen, char* buf);
+int base16384_decode(const char* data, int dlen, char* buf);
 base16384_err_t base16384_encode_file(const char* input, const char* output, char* encbuf, char* decbuf);
 base16384_err_t base16384_decode_file(const char* input, const char* output, char* encbuf, char* decbuf);
 
@@ -96,7 +96,7 @@
 """
 
 ffibuilder.set_source(
-    "pybase16384.backends.cffi._core_cffi",
+    "pybase16384.backends.cffi._core",
     source,
     sources=[f"./base16384/base14{CPUBIT}.c", "./base16384/file.c"],
     include_dirs=["./base16384"],

pybase16384/backends/cython/__init__.py

@@ -1,11 +1,13 @@
 """
 Copyright (c) 2008-2021 synodriver <synodriver@gmail.com>
 """
-from pybase16384.backends.cython._core_cy import (
+from pybase16384.backends.cython._core import (
     _decode,
     _decode_into,
+    _decode_parallel,
     _encode,
     _encode_into,
+    _encode_parallel,
     decode_fd,
     decode_file,
     decode_len,