Support the free-threaded build (#443)

See
#406 (comment).

It may be a little presumptuous to add wheel builds already but it looks
like that's how CI is set up in this repo. It may also make sense to add
some more tests that use either `rayon` or the python `threading`
module. That said, IMO you'd be testing guarantees provided by PyO3 more
than tiktoken itself.

Author: ngoldbaum

.github/workflows/build_wheels.yml

@@ -17,14 +17,15 @@ jobs:
         # cibuildwheel builds linux wheels inside a manylinux container
         # it also takes care of procuring the correct python version for us
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [39, 310, 311, 312, 313]
+        python-version: [39, 310, 311, 312, 313, 313t, 314, 314t]
 
     steps:
       - uses: actions/checkout@v4
 
-      - uses: pypa/cibuildwheel@v2.22.0
+      - uses: pypa/cibuildwheel@v3.1.4
         env:
           CIBW_BUILD: "cp${{ matrix.python-version}}-*"
+          CIBW_ENABLE: cpython-freethreading
 
       - uses: actions/upload-artifact@v4
         with:
@@ -39,19 +40,20 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-24.04-arm]
-        python-version: [39, 310, 311, 312, 313]
+        python-version: [39, 310, 311, 312, 313, 313t, 314, 314t]
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v3.1.4
         env:
           CIBW_BUILD: "cp${{ matrix.python-version}}-*"
           CIBW_ARCHS: aarch64
           CIBW_BUILD_VERBOSITY: 3
           # https://github.com/rust-lang/cargo/issues/10583
           CIBW_ENVIRONMENT_LINUX: PATH="$PATH:$HOME/.cargo/bin" CARGO_NET_GIT_FETCH_WITH_CLI=true
+          CIBW_ENABLE: cpython-freethreading
 
       - uses: actions/upload-artifact@v4
         with:

src/lib.rs

@@ -187,7 +187,7 @@ impl std::error::Error for EncodeError {}
 
 const MAX_NUM_THREADS: usize = 128;
 
-#[cfg_attr(feature = "python", pyclass)]
+#[cfg_attr(feature = "python", pyclass(frozen))]
 #[derive(Clone)]
 pub struct CoreBPE {
     encoder: HashMap<Vec<u8>, Rank>,

src/py.rs

@@ -183,7 +183,7 @@ impl CoreBPE {
     }
 }
 
-#[pyclass]
+#[pyclass(frozen)]
 struct TiktokenBuffer {
     tokens: Vec<Rank>,
 }
@@ -248,7 +248,7 @@ impl TiktokenBuffer {
     }
 }
 
-#[pymodule]
+#[pymodule(gil_used = false)]
 fn _tiktoken(_py: Python, m: &Bound<PyModule>) -> PyResult<()> {
     m.add_class::<CoreBPE>()?;
     Ok(())