-
Notifications
You must be signed in to change notification settings - Fork 138
Make ament_python_install_package() install a flat Python egg #316
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
8e38faf
Make ament_python_install_package() install a flat Python .egg
hidmic 4c518ea
Address peer review comments.
hidmic 1589dfb
Include subpackages and data files.
hidmic b3d783d
Fix package glob expressions
hidmic 57af8f5
Always use relative paths
hidmic a55116e
Normalize setuptools.find_packages() path
hidmic 7d8cc5c
Add a few missing quotes
hidmic 7876939
Rework how package_data is collected.
hidmic a2f1349
Add tests for find_packages_data()
hidmic d6d13a2
Fix find_packages_data() tests on Windows.
hidmic e7be7d4
Test byte-compiled Python sources are excluded.
hidmic 4db9bb8
Ignore byte-compiled Python files
hidmic 54aeedb
Add test for nested Python package in nested directory
hidmic File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| # Byte-compiled / optimized / DLL files | ||
| __pycache__/ | ||
| *.py[cod] |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| # Copyright 2021 Open Source Robotics Foundation, Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import fnmatch | ||
| import pathlib | ||
| from urllib.parse import urlparse | ||
|
|
||
| from setuptools import find_packages | ||
|
|
||
|
|
||
| def fuzzy_lookup(key, mapping): | ||
| """Lookup key in a mapping where keys may contain wildcards ('*').""" | ||
| for pattern, value in mapping.items(): | ||
| if fnmatch.fnmatch(key, pattern): | ||
| yield value | ||
|
|
||
|
|
||
| def find_packages_data(where='.', exclude=(), include=('*',)): | ||
| """ | ||
| Find data in Python packages found within directory 'where'. | ||
|
|
||
| Similar to `setuptools.find_packages`. | ||
|
|
||
| :param where: a cross-platform (i.e. URL-style) path | ||
| :param exclude: a dictionary that maps from package names to | ||
| lists of glob patterns to be excluded from the search. Wildcards | ||
| ('*') may be used in package names. A collection of package names | ||
| may be provided instead of a dictionary, in which case whole packages | ||
| will be excluded. | ||
| :param include: a dictionary that maps from package names to | ||
| lists of glob patterns to be included in the search. Wildcards | ||
| ('*') may be used in package names. A collection of package names | ||
| may be provided instead of a dictionary, in which case whole packages | ||
| will be included but excluding Python sources and byte-compiled code. | ||
| :returns: a dictionary suitable to be used as 'package_data' when calling | ||
| `setuptools.setup` | ||
| """ | ||
| packages = find_packages( | ||
| where=where, include=set(include), | ||
| # Defer package exclusion (may be partial) | ||
| ) | ||
| where = pathlib.Path(urlparse(where).path) | ||
| if not isinstance(exclude, dict): | ||
| # Exclude whole packages | ||
| exclude = {name: ['**/*'] for name in exclude} | ||
| if not isinstance(include, dict): | ||
| # Include whole packages | ||
| include = {name: ['**/*'] for name in include} | ||
| # But | ||
| for name in include: | ||
| # Exclude Python sources and byte-compiled code | ||
| if name not in exclude: | ||
| exclude[name] = [] | ||
| exclude[name].extend([ | ||
| '**/*.py', '**/*.pyc', | ||
| '**/__pycache__/**/*' | ||
| ]) | ||
|
|
||
| packages_data = {} | ||
| processed_data = set() | ||
| # Bottom-up search for packages' data | ||
| for name in sorted(packages, reverse=True): | ||
| rootpath = where / name.replace('.', '/') | ||
sloretz marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| # Exclude nested packages' content too | ||
| excluded_data = set(processed_data) | ||
| for patterns in fuzzy_lookup(name, exclude): | ||
| excluded_data.update( | ||
| path for pattern in patterns | ||
| for path in rootpath.glob(pattern) | ||
| ) | ||
|
|
||
| included_data = set() | ||
| for patterns in fuzzy_lookup(name, include): | ||
| included_data.update( | ||
| path for pattern in patterns | ||
| for path in rootpath.glob(pattern) | ||
| if not path.is_dir() and path not in excluded_data | ||
| ) | ||
|
|
||
| if included_data: | ||
| packages_data[name] = [ | ||
| str(path.relative_to(rootpath)) | ||
| for path in included_data | ||
| ] | ||
|
|
||
| # Keep track of packages processed | ||
| processed_data.update(rootpath.glob('**/*')) | ||
| processed_data.add(rootpath) | ||
|
|
||
| return packages_data | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
Empty file.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| # Copyright 2021 Open Source Robotics Foundation, Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import os | ||
| import unittest | ||
|
|
||
| from ament_cmake_python import find_packages_data | ||
|
|
||
|
|
||
| class TestFindPackagesData(unittest.TestCase): | ||
|
|
||
| def test_all_packages_data_is_found(self): | ||
| data = find_packages_data() | ||
| assert set(data) == {'foo', 'foo.bar', 'baz'} | ||
| assert set(data['foo']) == {'data', 'data.txt'} | ||
| assert set(data['foo.bar']) == { | ||
| 'data.txt', | ||
| os.path.join('resources', 'fizz.txt'), | ||
| os.path.join('resources', 'buzz.txt') | ||
| } | ||
| assert set(data['baz']) == {'data.bin', 'data'} | ||
|
|
||
| def test_whole_package_data_is_included(self): | ||
| data = find_packages_data( | ||
| include=('foo', 'foo.*')) | ||
| assert set(data) == {'foo', 'foo.bar'} | ||
| assert set(data['foo']) == {'data', 'data.txt'} | ||
| assert set(data['foo.bar']) == { | ||
| 'data.txt', | ||
| os.path.join('resources', 'fizz.txt'), | ||
| os.path.join('resources', 'buzz.txt') | ||
| } | ||
|
|
||
| def test_whole_package_data_is_excluded(self): | ||
| data = find_packages_data( | ||
| include=('foo', 'foo.*'), | ||
| exclude=('foo.bar',)) | ||
| assert set(data) == {'foo'} | ||
| assert set(data['foo']) == {'data', 'data.txt'} | ||
|
|
||
| def test_partial_package_data_is_excluded(self): | ||
| data = find_packages_data( | ||
| include=('foo', 'foo.*'), | ||
| exclude={'foo.bar': ['resources/*']}) | ||
| assert set(data) == {'foo', 'foo.bar'} | ||
| assert set(data['foo']) == {'data', 'data.txt'} | ||
| assert set(data['foo.bar']) == {'data.txt'} | ||
|
|
||
| def test_partial_package_data_is_included(self): | ||
| data = find_packages_data( | ||
| include={ | ||
| 'foo': ['*.txt'], | ||
| 'foo.*': ['resources/*.txt'] | ||
| }, | ||
| ) | ||
| assert set(data) == {'foo', 'foo.bar'} | ||
| assert set(data['foo']) == {'data.txt'} | ||
| assert set(data['foo.bar']) == { | ||
| os.path.join('resources', 'fizz.txt'), | ||
| os.path.join('resources', 'buzz.txt') | ||
| } | ||
|
|
||
| def test_nested_packages_data_is_found(self): | ||
| data = find_packages_data(where='nested/pkgs') | ||
| assert set(data) == {'fizz', 'fizz.buzz'} | ||
| assert set(data['fizz']) == { | ||
| os.path.join('data', 'buzz.bin') | ||
| } | ||
| assert set(data['fizz.buzz']) == {'data.txt'} | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| unittest.main() |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.