Sync codebase

Author: hauntsaninja

README.md

@@ -93,12 +93,12 @@ from setuptools import setup, find_namespace_packages
 
 setup(
     name="my_tiktoken_extension",
-    packages=find_namespace_packages(include=['tiktoken_ext.*'])
+    packages=find_namespace_packages(include=['tiktoken_ext*']),
     install_requires=["tiktoken"],
     ...
 )
 ```
 
-Then simply `pip install my_tiktoken_extension` and you should be able to use your custom encodings!
-Make sure **not** to use an editable install.
+Then simply `pip install ./my_tiktoken_extension` and you should be able to use your
+custom encodings! Make sure **not** to use an editable install.
 

pyproject.toml

@@ -1,9 +1,18 @@
 [project]
 name = "tiktoken"
 version = "0.2.0"
+description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models"
+readme = "README.md"
+license = {file = "LICENSE"}
+authors = [{name = "Shantanu Jain"}, {email = "[email protected]"}]
 dependencies = ["blobfile>=2", "regex>=2022.1.18", "requests>=2.26.0"]
 requires-python = ">=3.8"
 
+[project.urls]
+homepage = "https://github.com/openai/tiktoken"
+repository = "https://github.com/openai/tiktoken"
+changelog = "https://github.com/openai/tiktoken/blob/main/CHANGELOG.md"
+
 [build-system]
 build-backend = "setuptools.build_meta"
 requires = ["setuptools>=62.4", "wheel", "setuptools-rust>=1.5.2"]