diff options
author | Frank Feng <frankfeng@google.com> | 2022-06-09 22:56:10 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2022-06-09 22:56:10 +0000 |
commit | c5e9dc20126dbd2933ef8d9ba3707d526e60371b (patch) | |
tree | 1ba4dee98dadba3276d42ef41914625e9552a0f0 | |
parent | 8eebef1e74965fcdea069dfa5e828f67f2b9912d (diff) | |
parent | 3db5a4cd40152cbdddffa0107e408cddf9d3eccd (diff) | |
download | portpicker-c5e9dc20126dbd2933ef8d9ba3707d526e60371b.tar.gz |
Merge remote-tracking branch 'goog/mirror-aosp-master' into bp_portpicker am: 5f968166b2 am: 3db5a4cd40android-13.0.0_r83android-13.0.0_r82android-13.0.0_r81android-13.0.0_r80android-13.0.0_r79android-13.0.0_r78android-13.0.0_r77android-13.0.0_r76android-13.0.0_r75android-13.0.0_r74android-13.0.0_r73android-13.0.0_r72android-13.0.0_r71android-13.0.0_r70android-13.0.0_r69android-13.0.0_r68android-13.0.0_r67android-13.0.0_r66android-13.0.0_r65android-13.0.0_r64android-13.0.0_r63android-13.0.0_r62android-13.0.0_r61android-13.0.0_r60android-13.0.0_r59android-13.0.0_r58android-13.0.0_r56android-13.0.0_r54android-13.0.0_r53android-13.0.0_r52android-13.0.0_r51android-13.0.0_r50android-13.0.0_r49android-13.0.0_r48android-13.0.0_r47android-13.0.0_r46android-13.0.0_r45android-13.0.0_r44android-13.0.0_r43android-13.0.0_r42android-13.0.0_r41android-13.0.0_r40android-13.0.0_r39android-13.0.0_r38android-13.0.0_r37android-13.0.0_r36android-13.0.0_r35android-13.0.0_r34android-13.0.0_r33android-13.0.0_r32android13-qpr3-s9-releaseandroid13-qpr3-s8-releaseandroid13-qpr3-s7-releaseandroid13-qpr3-s6-releaseandroid13-qpr3-s5-releaseandroid13-qpr3-s4-releaseandroid13-qpr3-s3-releaseandroid13-qpr3-s2-releaseandroid13-qpr3-s14-releaseandroid13-qpr3-s13-releaseandroid13-qpr3-s12-releaseandroid13-qpr3-s11-releaseandroid13-qpr3-s10-releaseandroid13-qpr3-s1-releaseandroid13-qpr3-releaseandroid13-qpr3-c-s8-releaseandroid13-qpr3-c-s7-releaseandroid13-qpr3-c-s6-releaseandroid13-qpr3-c-s5-releaseandroid13-qpr3-c-s4-releaseandroid13-qpr3-c-s3-releaseandroid13-qpr3-c-s2-releaseandroid13-qpr3-c-s12-releaseandroid13-qpr3-c-s11-releaseandroid13-qpr3-c-s10-releaseandroid13-qpr3-c-s1-releaseandroid13-qpr2-s9-releaseandroid13-qpr2-s8-releaseandroid13-qpr2-s7-releaseandroid13-qpr2-s6-releaseandroid13-qpr2-s5-releaseandroid13-qpr2-s3-releaseandroid13-qpr2-s2-releaseandroid13-qpr2-s12-releaseandroid13-qpr2-s11-releaseandroid13-qpr2-s10-releaseandroid13-qpr2-s1-releaseandroid13-qpr2-releaseandroid13-qpr2-b-s1-releaseandroid13-d4-s2-releaseandroid13-d4-s1-releaseandroid13-d4-release
Original change: https://googleplex-android-review.googlesource.com/c/platform/external/python/portpicker/+/18820206
Change-Id: I29124fdfedf80c73c1feb6703f4f1b97dd83fcdd
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
-rw-r--r-- | .github/workflows/python-package.yml | 61 | ||||
-rw-r--r-- | .gitignore | 7 | ||||
-rw-r--r-- | .travis.yml | 15 | ||||
-rw-r--r-- | Android.bp | 28 | ||||
-rw-r--r-- | CONTRIBUTING.md | 26 | ||||
-rw-r--r-- | ChangeLog.md | 65 | ||||
-rw-r--r-- | LICENSE | 176 | ||||
-rw-r--r-- | MANIFEST.in | 9 | ||||
-rw-r--r-- | METADATA | 16 | ||||
-rw-r--r-- | MODULE_LICENSE_APACHE2 | 0 | ||||
-rw-r--r-- | NOTICE | 176 | ||||
-rw-r--r-- | OWNERS | 8 | ||||
-rw-r--r-- | README.md | 66 | ||||
-rwxr-xr-x | package.sh | 11 | ||||
-rw-r--r-- | pyproject.toml | 21 | ||||
-rw-r--r-- | setup.cfg | 41 | ||||
-rw-r--r-- | src/Android.bp | 35 | ||||
-rw-r--r-- | src/__init__.py | 335 | ||||
-rw-r--r-- | src/portpicker.py | 335 | ||||
-rw-r--r-- | src/portserver.py | 415 | ||||
-rw-r--r-- | src/tests/portpicker_test.py | 390 | ||||
-rw-r--r-- | src/tests/portserver_test.py | 370 | ||||
-rwxr-xr-x | test.sh | 12 |
23 files changed, 2618 insertions, 0 deletions
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000..3ee553d --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,61 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python Portpicker & Portserver + +on: + push: + branches: + - 'main' + pull_request: + branches: + - 'main' + +jobs: + build-ubuntu: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest tox + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Test with tox + run: | + # Run tox using the version of Python in `PATH` + tox -e py + + build-windows: + + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest tox + if (Test-Path "requirements.txt") { pip install -r requirements.txt } + - name: Test with tox + run: | + # Run tox using the version of Python in `PATH` + tox -e py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8ce580d --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +*.pyc +__pycache__ +build +dist +MANIFEST +.tox +portpicker.egg-info diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5c5e2ad --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +language: python +python: + - "3.6" + - "3.7" + - "3.8" + - "3.9" + - "3.10-dev" +os: linux +arch: + - ppc64le +dist: focal +install: + - pip install --upgrade pip + - pip install tox-travis +script: tox diff --git a/Android.bp b/Android.bp new file mode 100644 index 0000000..eb55539 --- /dev/null +++ b/Android.bp @@ -0,0 +1,28 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package { + default_applicable_licenses: ["external_python_portpicker_license"], +} + +license { + name: "external_python_portpicker_license", + visibility: [":__subpackages__"], + license_kinds: [ + "SPDX-license-identifier-Apache-2.0", + ], + license_text: [ + "LICENSE", + ], +} + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..13608e3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# How To Contribute + +Want to contribute? Great! First, read this page (including the small print at the end). + +### Before you contribute +Before we can use your code, you must sign the +[Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1) +(CLA), which you can do online. The CLA is necessary mainly because you own the +copyright to your changes, even after your contribution becomes part of our +codebase, so we need your permission to use and distribute your code. We also +need to be sure of various other things—for instance that you'll tell us if you +know that your code infringes on other people's patents. You don't have to sign +the CLA until after you've submitted your code for review and a member has +approved it, but you must do it before we can put your code into our codebase. +Before you start working on a larger contribution, you should get in touch with +us first through the issue tracker with your idea so that we can help out and +possibly guide you. Coordinating up front makes it much easier to avoid +frustration later on. + +### Code reviews +All submissions, including submissions by project members, require review. We +use Github pull requests for this purpose. + +### The small print +Contributions made by corporations are covered by a different agreement than +the one above, the Software Grant and Corporate Contributor License Agreement. diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 0000000..3cda728 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,65 @@ +## 1.5.1 + +* When not using a portserver *(you really should)*, try the `bind(0)` + approach before hunting for random unused ports. More reliable per + https://github.com/google/python_portpicker/issues/16. + +## 1.5.0 + +* Add portserver support to Windows using named pipes. To create or connect to + a server, prefix the name of the server with `@` (e.g. + `@unittest-portserver`). + +## 1.4.0 + +* Use `async def` instead of `@asyncio.coroutine` in order to support 3.10. +* The portserver now checks for and rejects pid values that are out of range. +* Declare a minimum Python version of 3.6 in the package config. +* Rework `portserver_test.py` to launch an actual portserver process instead + of mocks. + +## 1.3.9 + +* No portpicker or portserver code changes +* Fixed the portserver test on recent Python 3.x versions. +* Switched to setup.cfg based packaging. +* We no longer declare ourselves Python 2.7 or 3.3-3.5 compatible. + +## 1.3.1 + +* Fix a race condition in `pick_unused_port()` involving the free ports set. + +## 1.3.0 + +* Adds an optional `portserver_address` parameter to `pick_unused_port()` so + that callers can specify their own regardless of `os.environ`. +* `pick_unused_port()` now raises `NoFreePortFoundError` when no available + port could be found rather than spinning in a loop trying forever. +* Fall back to `socket.AF_INET` when `socket.AF_UNIX` support is not available + to communicate with a portserver. + +## 1.2.0 + +* Introduced `add_reserved_port()` and `return_port()` APIs to allow ports to + be recycled and allow users to bring ports of their own. + +## 1.1.1 + +* Changed default port range to 15000-24999 to avoid ephemeral ports. +* Portserver bugfix. + +## 1.1.0 + +* Renamed portpicker APIs to use PEP8 style function names in code and docs. +* Legacy CapWords API name compatibility is maintained (and explicitly + tested). + +## 1.0.1 + +* Code reindented to use 4 space indents and run through + [YAPF](https://github.com/google/yapf) for consistent style. +* Not packaged for release. + +## 1.0.0 + +* Original open source release. @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a5db4a8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +include src/port*.py +include src/tests/port*.py +include README.md +include LICENSE +include CONTRIBUTING.md +include ChangeLog.md +include setup.py +include test.sh +exclude package.sh diff --git a/METADATA b/METADATA new file mode 100644 index 0000000..8480a2c --- /dev/null +++ b/METADATA @@ -0,0 +1,16 @@ +name: "python_portpicker" +description: + "This module is useful for finding unused network ports on a host." +third_party { + url { + type: HOMEPAGE + value: "https://github.com/google/python_portpicker" + } + url { + type: GIT + value: "https://github.com/google/python_portpicker" + } + version: "b05ca660bc9ce2ff9753256238927b91e234c34b" + last_upgrade_date { year: 2022 month: 5 day: 17 } + license_type: NOTICE +} diff --git a/MODULE_LICENSE_APACHE2 b/MODULE_LICENSE_APACHE2 new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/MODULE_LICENSE_APACHE2 @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS @@ -0,0 +1,8 @@ +# Android side engprod team +jdesprez@google.com +frankfeng@google.com +murj@google.com + +# Mobly team - use for mobly bugs +angli@google.com +lancefluger@google.com diff --git a/README.md b/README.md new file mode 100644 index 0000000..bd56703 --- /dev/null +++ b/README.md @@ -0,0 +1,66 @@ +# Python portpicker module + +[![PyPI version](https://badge.fury.io/py/portpicker.svg)](https://badge.fury.io/py/portpicker) +![GH Action Status](https://github.com/google/python_portpicker/actions/workflows/python-package.yml/badge.svg) +[![Travis CI org Status](https://travis-ci.org/google/python_portpicker.svg?branch=master)](https://travis-ci.org/google/python_portpicker) + +This module is useful for finding unused network ports on a host. If you need +legacy Python 2 support, use the 1.3.x releases. + +This module provides a pure Python `pick_unused_port()` function. It can also be +called via the command line for use in shell scripts. + +If your code can accept a bound TCP socket rather than a port number consider +using `socket.bind(('localhost', 0))` to bind atomically to an available port +rather than using this library at all. + +There is a race condition between picking a port and your application code +binding to it. The use of a port server by all of your test code to avoid that +problem is recommended on loaded test hosts running many tests at a time. + +Unless you are using a port server, subsequent calls to `pick_unused_port()` to +obtain an additional port are not guaranteed to return a unique port. + +### What is the optional port server? + +A port server is intended to be run as a daemon, for use by all processes +running on the host. It coordinates uses of network ports by anything using a +portpicker library. If you are using hosts as part of a test automation cluster, +each one should run a port server as a daemon. You should set the +`PORTSERVER_ADDRESS=@unittest-portserver` environment variable on all of your +test runners so that portpicker makes use of it. + +A sample port server is included. This portserver implementation works but has +not spent time in production. If you use it with good results please report back +so that this statement can be updated to reflect that. :) + +A port server listens on a unix socket, reads a pid from a new connection, tests +the ports it is managing and replies with a port assignment port for that pid. A +port is only reclaimed for potential reassignment to another process after the +process it was originally assigned to has died. Processes that need multiple +ports can simply issue multiple requests and are guaranteed they will each be +unique. + +## Typical usage: + +```python +import portpicker +test_port = portpicker.pick_unused_port() +``` + +Or from the command line: + +```bash +TEST_PORT=`/path/to/portpicker.py $$` +``` + +Or, if portpicker is installed as a library on the system Python interpreter: + +```bash +TEST_PORT=`python3 -m portpicker $$` +``` + +## DISCLAIMER + +This is not an official Google product (experimental or otherwise), it is just +code that happens to be owned by Google. diff --git a/package.sh b/package.sh new file mode 100755 index 0000000..7fb24ad --- /dev/null +++ b/package.sh @@ -0,0 +1,11 @@ +#!/bin/sh -ex + +unset PYTHONPATH +python3 -m venv build/venv +. build/venv/bin/activate + +pip install --upgrade build twine +python -m build +twine check dist/* + +echo 'When ready, upload to PyPI using: build/venv/bin/twine upload dist/*' diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b1236df --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools >= 40.9.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.tox] +legacy_tox_ini = """ +[tox] +envlist = py{36,37,38,39} +isolated_build = true +skip_missing_interpreters = true +# minimum tox version +minversion = 3.3.0 +[testenv] +deps = + check-manifest >= 0.42 + pytest +commands = + check-manifest --ignore 'src/tests/**' + python -c 'from setuptools import setup; setup()' check -m -s + py.test -s {posargs} +""" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..63c1ac4 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,41 @@ +# https://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files +[metadata] +name = portpicker +version = 1.5.1b1 +maintainer = Google LLC +maintainer_email = greg@krypto.org +license = Apache 2.0 +license_files = LICENSE +description = A library to choose unique available network ports. +url = https://github.com/google/python_portpicker +long_description = Portpicker provides an API to find and return an available + network port for an application to bind to. Ideally suited for use from + unittests or for test harnesses that launch local servers. + + It also contains an optional portserver that can be used to coordinate + allocation of network ports on a single build/test farm host across all + processes willing to use a port server aware port picker library such as + this one. +classifiers = + Development Status :: 5 - Production/Stable + License :: OSI Approved :: Apache Software License + Intended Audience :: Developers + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy +platforms = POSIX, Windows +requires = + +[options] +install_requires = psutil +python_requires = >= 3.6 +package_dir= + =src +py_modules = portpicker +scripts = src/portserver.py diff --git a/src/Android.bp b/src/Android.bp new file mode 100644 index 0000000..0d5b8ac --- /dev/null +++ b/src/Android.bp @@ -0,0 +1,35 @@ +// Copyright 2022 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package { + default_applicable_licenses: ["external_python_portpicker_license"], +} + +python_library { + name: "py-portpicker", + host_supported: true, + srcs: [ + "__init__.py", + "portpicker.py", + "portserver.py", + ], + version: { + py2: { + enabled: false, + }, + py3: { + enabled: true, + }, + }, + pkg_path: "portpicker", +} diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..fc2825b --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,335 @@ +#!/usr/bin/python3 +# +# Copyright 2007 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Pure python code for finding unused ports on a host. + +This module provides a pick_unused_port() function. +It can also be called via the command line for use in shell scripts. +When called from the command line, it takes one optional argument, which, +if given, is sent to portserver instead of portpicker's PID. +To reserve a port for the lifetime of a bash script, use $BASHPID as this +argument. + +There is a race condition between picking a port and your application code +binding to it. The use of a port server to prevent that is recommended on +loaded test hosts running many tests at a time. + +If your code can accept a bound socket as input rather than being handed a +port number consider using socket.bind(('localhost', 0)) to bind to an +available port without a race condition rather than using this library. + +Typical usage: + test_port = portpicker.pick_unused_port() +""" + +from __future__ import print_function + +import logging +import os +import random +import socket +import sys + +if sys.platform == 'win32': + import _winapi +else: + _winapi = None + +# The legacy Bind, IsPortFree, etc. names are not exported. +__all__ = ('bind', 'is_port_free', 'pick_unused_port', 'return_port', + 'add_reserved_port', 'get_port_from_port_server') + +_PROTOS = [(socket.SOCK_STREAM, socket.IPPROTO_TCP), + (socket.SOCK_DGRAM, socket.IPPROTO_UDP)] + + +# Ports that are currently available to be given out. +_free_ports = set() + +# Ports that are reserved or from the portserver that may be returned. +_owned_ports = set() + +# Ports that we chose randomly that may be returned. +_random_ports = set() + + +class NoFreePortFoundError(Exception): + """Exception indicating that no free port could be found.""" + + +def add_reserved_port(port): + """Add a port that was acquired by means other than the port server.""" + _free_ports.add(port) + + +def return_port(port): + """Return a port that is no longer being used so it can be reused.""" + if port in _random_ports: + _random_ports.remove(port) + elif port in _owned_ports: + _owned_ports.remove(port) + _free_ports.add(port) + elif port in _free_ports: + logging.info("Returning a port that was already returned: %s", port) + else: + logging.info("Returning a port that wasn't given by portpicker: %s", + port) + + +def bind(port, socket_type, socket_proto): + """Try to bind to a socket of the specified type, protocol, and port. + + This is primarily a helper function for PickUnusedPort, used to see + if a particular port number is available. + + For the port to be considered available, the kernel must support at least + one of (IPv6, IPv4), and the port must be available on each supported + family. + + Args: + port: The port number to bind to, or 0 to have the OS pick a free port. + socket_type: The type of the socket (ex: socket.SOCK_STREAM). + socket_proto: The protocol of the socket (ex: socket.IPPROTO_TCP). + + Returns: + The port number on success or None on failure. + """ + got_socket = False + for family in (socket.AF_INET6, socket.AF_INET): + try: + sock = socket.socket(family, socket_type, socket_proto) + got_socket = True + except socket.error: + continue + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('', port)) + if socket_type == socket.SOCK_STREAM: + sock.listen(1) + port = sock.getsockname()[1] + except socket.error: + return None + finally: + sock.close() + return port if got_socket else None + +Bind = bind # legacy API. pylint: disable=invalid-name + + +def is_port_free(port): + """Check if specified port is free. + + Args: + port: integer, port to check + Returns: + boolean, whether it is free to use for both TCP and UDP + """ + return bind(port, *_PROTOS[0]) and bind(port, *_PROTOS[1]) + +IsPortFree = is_port_free # legacy API. pylint: disable=invalid-name + + +def pick_unused_port(pid=None, portserver_address=None): + """A pure python implementation of PickUnusedPort. + + Args: + pid: PID to tell the portserver to associate the reservation with. If + None, the current process's PID is used. + portserver_address: The address (path) of a unix domain socket + with which to connect to a portserver, a leading '@' + character indicates an address in the "abstract namespace". OR + On systems without socket.AF_UNIX, this is an AF_INET address. + If None, or no port is returned by the portserver at the provided + address, the environment will be checked for a PORTSERVER_ADDRESS + variable. If that is not set, no port server will be used. + + Returns: + A port number that is unused on both TCP and UDP. + + Raises: + NoFreePortFoundError: No free port could be found. + """ + try: # Instead of `if _free_ports:` to handle the race condition. + port = _free_ports.pop() + except KeyError: + pass + else: + _owned_ports.add(port) + return port + # Provide access to the portserver on an opt-in basis. + if portserver_address: + port = get_port_from_port_server(portserver_address, pid=pid) + if port: + return port + if 'PORTSERVER_ADDRESS' in os.environ: + port = get_port_from_port_server(os.environ['PORTSERVER_ADDRESS'], + pid=pid) + if port: + return port + return _pick_unused_port_without_server() + +PickUnusedPort = pick_unused_port # legacy API. pylint: disable=invalid-name + + +def _pick_unused_port_without_server(): # Protected. pylint: disable=invalid-name + """Pick an available network port without the help of a port server. + + This code ensures that the port is available on both TCP and UDP. + + This function is an implementation detail of PickUnusedPort(), and + should not be called by code outside of this module. + + Returns: + A port number that is unused on both TCP and UDP. + + Raises: + NoFreePortFoundError: No free port could be found. + """ + # Next, try a few times to get an OS-assigned port. + # Ambrose discovered that on the 2.6 kernel, calling Bind() on UDP socket + # returns the same port over and over. So always try TCP first. + for _ in range(10): + # Ask the OS for an unused port. + port = bind(0, _PROTOS[0][0], _PROTOS[0][1]) + # Check if this port is unused on the other protocol. + if port and bind(port, _PROTOS[1][0], _PROTOS[1][1]): + _random_ports.add(port) + return port + + # Try random ports as a last resort. + rng = random.Random() + for _ in range(10): + port = int(rng.randrange(15000, 25000)) + if is_port_free(port): + _random_ports.add(port) + return port + + # Give up. + raise NoFreePortFoundError() + + +def _get_linux_port_from_port_server(portserver_address, pid): + # An AF_UNIX address may start with a zero byte, in which case it is in the + # "abstract namespace", and doesn't have any filesystem representation. + # See 'man 7 unix' for details. + # The convention is to write '@' in the address to represent this zero byte. + if portserver_address[0] == '@': + portserver_address = '\0' + portserver_address[1:] + + try: + # Create socket. + if hasattr(socket, 'AF_UNIX'): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # pylint: disable=no-member + else: + # fallback to AF_INET if this is not unix + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + # Connect to portserver. + sock.connect(portserver_address) + + # Write request. + sock.sendall(('%d\n' % pid).encode('ascii')) + + # Read response. + # 1K should be ample buffer space. + return sock.recv(1024) + finally: + sock.close() + except socket.error as error: + print('Socket error when connecting to portserver:', error, + file=sys.stderr) + return None + + +def _get_windows_port_from_port_server(portserver_address, pid): + if portserver_address[0] == '@': + portserver_address = '\\\\.\\pipe\\' + portserver_address[1:] + + try: + handle = _winapi.CreateFile( + portserver_address, + _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, + 0, + 0, + _winapi.OPEN_EXISTING, + 0, + 0) + + _winapi.WriteFile(handle, ('%d\n' % pid).encode('ascii')) + data, _ = _winapi.ReadFile(handle, 6, 0) + return data + except FileNotFoundError as error: + print('File error when connecting to portserver:', error, + file=sys.stderr) + return None + +def get_port_from_port_server(portserver_address, pid=None): + """Request a free a port from a system-wide portserver. + + This follows a very simple portserver protocol: + The request consists of our pid (in ASCII) followed by a newline. + The response is a port number and a newline, 0 on failure. + + This function is an implementation detail of pick_unused_port(). + It should not normally be called by code outside of this module. + + Args: + portserver_address: The address (path) of a unix domain socket + with which to connect to the portserver. A leading '@' + character indicates an address in the "abstract namespace." + On systems without socket.AF_UNIX, this is an AF_INET address. + pid: The PID to tell the portserver to associate the reservation with. + If None, the current process's PID is used. + + Returns: + The port number on success or None on failure. + """ + if not portserver_address: + return None + + if pid is None: + pid = os.getpid() + + if _winapi: + buf = _get_windows_port_from_port_server(portserver_address, pid) + else: + buf = _get_linux_port_from_port_server(portserver_address, pid) + + if buf is None: + return None + + try: + port = int(buf.split(b'\n')[0]) + except ValueError: + print('Portserver failed to find a port.', file=sys.stderr) + return None + _owned_ports.add(port) + return port + + +GetPortFromPortServer = get_port_from_port_server # legacy API. pylint: disable=invalid-name + + +def main(argv): + """If passed an arg, treat it as a PID, otherwise portpicker uses getpid.""" + port = pick_unused_port(pid=int(argv[1]) if len(argv) > 1 else None) + if not port: + sys.exit(1) + print(port) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/src/portpicker.py b/src/portpicker.py new file mode 100644 index 0000000..fc2825b --- /dev/null +++ b/src/portpicker.py @@ -0,0 +1,335 @@ +#!/usr/bin/python3 +# +# Copyright 2007 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Pure python code for finding unused ports on a host. + +This module provides a pick_unused_port() function. +It can also be called via the command line for use in shell scripts. +When called from the command line, it takes one optional argument, which, +if given, is sent to portserver instead of portpicker's PID. +To reserve a port for the lifetime of a bash script, use $BASHPID as this +argument. + +There is a race condition between picking a port and your application code +binding to it. The use of a port server to prevent that is recommended on +loaded test hosts running many tests at a time. + +If your code can accept a bound socket as input rather than being handed a +port number consider using socket.bind(('localhost', 0)) to bind to an +available port without a race condition rather than using this library. + +Typical usage: + test_port = portpicker.pick_unused_port() +""" + +from __future__ import print_function + +import logging +import os +import random +import socket +import sys + +if sys.platform == 'win32': + import _winapi +else: + _winapi = None + +# The legacy Bind, IsPortFree, etc. names are not exported. +__all__ = ('bind', 'is_port_free', 'pick_unused_port', 'return_port', + 'add_reserved_port', 'get_port_from_port_server') + +_PROTOS = [(socket.SOCK_STREAM, socket.IPPROTO_TCP), + (socket.SOCK_DGRAM, socket.IPPROTO_UDP)] + + +# Ports that are currently available to be given out. +_free_ports = set() + +# Ports that are reserved or from the portserver that may be returned. +_owned_ports = set() + +# Ports that we chose randomly that may be returned. +_random_ports = set() + + +class NoFreePortFoundError(Exception): + """Exception indicating that no free port could be found.""" + + +def add_reserved_port(port): + """Add a port that was acquired by means other than the port server.""" + _free_ports.add(port) + + +def return_port(port): + """Return a port that is no longer being used so it can be reused.""" + if port in _random_ports: + _random_ports.remove(port) + elif port in _owned_ports: + _owned_ports.remove(port) + _free_ports.add(port) + elif port in _free_ports: + logging.info("Returning a port that was already returned: %s", port) + else: + logging.info("Returning a port that wasn't given by portpicker: %s", + port) + + +def bind(port, socket_type, socket_proto): + """Try to bind to a socket of the specified type, protocol, and port. + + This is primarily a helper function for PickUnusedPort, used to see + if a particular port number is available. + + For the port to be considered available, the kernel must support at least + one of (IPv6, IPv4), and the port must be available on each supported + family. + + Args: + port: The port number to bind to, or 0 to have the OS pick a free port. + socket_type: The type of the socket (ex: socket.SOCK_STREAM). + socket_proto: The protocol of the socket (ex: socket.IPPROTO_TCP). + + Returns: + The port number on success or None on failure. + """ + got_socket = False + for family in (socket.AF_INET6, socket.AF_INET): + try: + sock = socket.socket(family, socket_type, socket_proto) + got_socket = True + except socket.error: + continue + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('', port)) + if socket_type == socket.SOCK_STREAM: + sock.listen(1) + port = sock.getsockname()[1] + except socket.error: + return None + finally: + sock.close() + return port if got_socket else None + +Bind = bind # legacy API. pylint: disable=invalid-name + + +def is_port_free(port): + """Check if specified port is free. + + Args: + port: integer, port to check + Returns: + boolean, whether it is free to use for both TCP and UDP + """ + return bind(port, *_PROTOS[0]) and bind(port, *_PROTOS[1]) + +IsPortFree = is_port_free # legacy API. pylint: disable=invalid-name + + +def pick_unused_port(pid=None, portserver_address=None): + """A pure python implementation of PickUnusedPort. + + Args: + pid: PID to tell the portserver to associate the reservation with. If + None, the current process's PID is used. + portserver_address: The address (path) of a unix domain socket + with which to connect to a portserver, a leading '@' + character indicates an address in the "abstract namespace". OR + On systems without socket.AF_UNIX, this is an AF_INET address. + If None, or no port is returned by the portserver at the provided + address, the environment will be checked for a PORTSERVER_ADDRESS + variable. If that is not set, no port server will be used. + + Returns: + A port number that is unused on both TCP and UDP. + + Raises: + NoFreePortFoundError: No free port could be found. + """ + try: # Instead of `if _free_ports:` to handle the race condition. + port = _free_ports.pop() + except KeyError: + pass + else: + _owned_ports.add(port) + return port + # Provide access to the portserver on an opt-in basis. + if portserver_address: + port = get_port_from_port_server(portserver_address, pid=pid) + if port: + return port + if 'PORTSERVER_ADDRESS' in os.environ: + port = get_port_from_port_server(os.environ['PORTSERVER_ADDRESS'], + pid=pid) + if port: + return port + return _pick_unused_port_without_server() + +PickUnusedPort = pick_unused_port # legacy API. pylint: disable=invalid-name + + +def _pick_unused_port_without_server(): # Protected. pylint: disable=invalid-name + """Pick an available network port without the help of a port server. + + This code ensures that the port is available on both TCP and UDP. + + This function is an implementation detail of PickUnusedPort(), and + should not be called by code outside of this module. + + Returns: + A port number that is unused on both TCP and UDP. + + Raises: + NoFreePortFoundError: No free port could be found. + """ + # Next, try a few times to get an OS-assigned port. + # Ambrose discovered that on the 2.6 kernel, calling Bind() on UDP socket + # returns the same port over and over. So always try TCP first. + for _ in range(10): + # Ask the OS for an unused port. + port = bind(0, _PROTOS[0][0], _PROTOS[0][1]) + # Check if this port is unused on the other protocol. + if port and bind(port, _PROTOS[1][0], _PROTOS[1][1]): + _random_ports.add(port) + return port + + # Try random ports as a last resort. + rng = random.Random() + for _ in range(10): + port = int(rng.randrange(15000, 25000)) + if is_port_free(port): + _random_ports.add(port) + return port + + # Give up. + raise NoFreePortFoundError() + + +def _get_linux_port_from_port_server(portserver_address, pid): + # An AF_UNIX address may start with a zero byte, in which case it is in the + # "abstract namespace", and doesn't have any filesystem representation. + # See 'man 7 unix' for details. + # The convention is to write '@' in the address to represent this zero byte. + if portserver_address[0] == '@': + portserver_address = '\0' + portserver_address[1:] + + try: + # Create socket. + if hasattr(socket, 'AF_UNIX'): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # pylint: disable=no-member + else: + # fallback to AF_INET if this is not unix + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + # Connect to portserver. + sock.connect(portserver_address) + + # Write request. + sock.sendall(('%d\n' % pid).encode('ascii')) + + # Read response. + # 1K should be ample buffer space. + return sock.recv(1024) + finally: + sock.close() + except socket.error as error: + print('Socket error when connecting to portserver:', error, + file=sys.stderr) + return None + + +def _get_windows_port_from_port_server(portserver_address, pid): + if portserver_address[0] == '@': + portserver_address = '\\\\.\\pipe\\' + portserver_address[1:] + + try: + handle = _winapi.CreateFile( + portserver_address, + _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, + 0, + 0, + _winapi.OPEN_EXISTING, + 0, + 0) + + _winapi.WriteFile(handle, ('%d\n' % pid).encode('ascii')) + data, _ = _winapi.ReadFile(handle, 6, 0) + return data + except FileNotFoundError as error: + print('File error when connecting to portserver:', error, + file=sys.stderr) + return None + +def get_port_from_port_server(portserver_address, pid=None): + """Request a free a port from a system-wide portserver. + + This follows a very simple portserver protocol: + The request consists of our pid (in ASCII) followed by a newline. + The response is a port number and a newline, 0 on failure. + + This function is an implementation detail of pick_unused_port(). + It should not normally be called by code outside of this module. + + Args: + portserver_address: The address (path) of a unix domain socket + with which to connect to the portserver. A leading '@' + character indicates an address in the "abstract namespace." + On systems without socket.AF_UNIX, this is an AF_INET address. + pid: The PID to tell the portserver to associate the reservation with. + If None, the current process's PID is used. + + Returns: + The port number on success or None on failure. + """ + if not portserver_address: + return None + + if pid is None: + pid = os.getpid() + + if _winapi: + buf = _get_windows_port_from_port_server(portserver_address, pid) + else: + buf = _get_linux_port_from_port_server(portserver_address, pid) + + if buf is None: + return None + + try: + port = int(buf.split(b'\n')[0]) + except ValueError: + print('Portserver failed to find a port.', file=sys.stderr) + return None + _owned_ports.add(port) + return port + + +GetPortFromPortServer = get_port_from_port_server # legacy API. pylint: disable=invalid-name + + +def main(argv): + """If passed an arg, treat it as a PID, otherwise portpicker uses getpid.""" + port = pick_unused_port(pid=int(argv[1]) if len(argv) > 1 else None) + if not port: + sys.exit(1) + print(port) + + +if __name__ == '__main__': + main(sys.argv) diff --git a/src/portserver.py b/src/portserver.py new file mode 100644 index 0000000..f986f3f --- /dev/null +++ b/src/portserver.py @@ -0,0 +1,415 @@ +#!/usr/bin/python3 +# +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""A server to hand out network ports to applications running on one host. + +Typical usage: + 1) Run one instance of this process on each of your unittest farm hosts. + 2) Set the PORTSERVER_ADDRESS environment variable in your test runner + environment to let the portpicker library know to use a port server + rather than attempt to find ports on its own. + +$ /path/to/portserver.py & +$ export PORTSERVER_ADDRESS=@unittest-portserver +$ # ... launch a bunch of unittest runners using portpicker ... +""" + +import argparse +import asyncio +import collections +import logging +import signal +import socket +import sys +import psutil +import subprocess +from datetime import datetime, timezone, timedelta + +log = None # Initialized to a logging.Logger by _configure_logging(). + +_PROTOS = [(socket.SOCK_STREAM, socket.IPPROTO_TCP), + (socket.SOCK_DGRAM, socket.IPPROTO_UDP)] + + +def _get_process_command_line(pid): + try: + return psutil.Process(pid).cmdline() + except psutil.NoSuchProcess: + return '' + + +def _get_process_start_time(pid): + try: + return psutil.Process(pid).create_time() + except psutil.NoSuchProcess: + return 0.0 + + +# TODO: Consider importing portpicker.bind() instead of duplicating the code. +def _bind(port, socket_type, socket_proto): + """Try to bind to a socket of the specified type, protocol, and port. + + For the port to be considered available, the kernel must support at least + one of (IPv6, IPv4), and the port must be available on each supported + family. + + Args: + port: The port number to bind to, or 0 to have the OS pick a free port. + socket_type: The type of the socket (ex: socket.SOCK_STREAM). + socket_proto: The protocol of the socket (ex: socket.IPPROTO_TCP). + + Returns: + The port number on success or None on failure. + """ + got_socket = False + for family in (socket.AF_INET6, socket.AF_INET): + try: + sock = socket.socket(family, socket_type, socket_proto) + got_socket = True + except socket.error: + continue + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('', port)) + if socket_type == socket.SOCK_STREAM: + sock.listen(1) + port = sock.getsockname()[1] + except socket.error: + return None + finally: + sock.close() + return port if got_socket else None + + +def _is_port_free(port): + """Check if specified port is free. + + Args: + port: integer, port to check + Returns: + boolean, whether it is free to use for both TCP and UDP + """ + return _bind(port, *_PROTOS[0]) and _bind(port, *_PROTOS[1]) + + +def _should_allocate_port(pid): + """Determine if we should allocate a port for use by the given process id.""" + if pid <= 0: + log.info('Not allocating a port to invalid pid') + return False + if pid == 1: + # The client probably meant to send us its parent pid but + # had been reparented to init. + log.info('Not allocating a port to init.') + return False + + if not psutil.pid_exists(pid): + log.info('Not allocating a port to a non-existent process') + return False + return True + + +async def _start_windows_server(client_connected_cb, path): + """Start the server on Windows using named pipes.""" + def protocol_factory(): + stream_reader = asyncio.StreamReader() + stream_reader_protocol = asyncio.StreamReaderProtocol( + stream_reader, client_connected_cb) + return stream_reader_protocol + + loop = asyncio.get_event_loop() + server, *_ = await loop.start_serving_pipe(protocol_factory, address=path) + + return server + + +class _PortInfo(object): + """Container class for information about a given port assignment. + + Attributes: + port: integer port number + pid: integer process id or 0 if unassigned. + start_time: Time in seconds since the epoch that the process started. + """ + + __slots__ = ('port', 'pid', 'start_time') + + def __init__(self, port): + self.port = port + self.pid = 0 + self.start_time = 0.0 + + +class _PortPool(object): + """Manage available ports for processes. + + Ports are reclaimed when the reserving process exits and the reserved port + is no longer in use. Only ports which are free for both TCP and UDP will be + handed out. It is easier to not differentiate between protocols. + + The pool must be pre-seeded with add_port_to_free_pool() calls + after which get_port_for_process() will allocate and reclaim ports. + The len() of a _PortPool returns the total number of ports being managed. + + Attributes: + ports_checked_for_last_request: The number of ports examined in order to + return from the most recent get_port_for_process() request. A high + number here likely means the number of available ports with no active + process using them is getting low. + """ + + def __init__(self): + self._port_queue = collections.deque() + self.ports_checked_for_last_request = 0 + + def num_ports(self): + return len(self._port_queue) + + def get_port_for_process(self, pid): + """Allocates and returns port for pid or 0 if none could be allocated.""" + if not self._port_queue: + raise RuntimeError('No ports being managed.') + + # Avoid an infinite loop if all ports are currently assigned. + check_count = 0 + max_ports_to_test = len(self._port_queue) + while check_count < max_ports_to_test: + # Get the next candidate port and move it to the back of the queue. + candidate = self._port_queue.pop() + self._port_queue.appendleft(candidate) + check_count += 1 + if (candidate.start_time == 0.0 or + candidate.start_time != _get_process_start_time(candidate.pid)): + if _is_port_free(candidate.port): + candidate.pid = pid + candidate.start_time = _get_process_start_time(pid) + if not candidate.start_time: + log.info("Can't read start time for pid %d.", pid) + self.ports_checked_for_last_request = check_count + return candidate.port + else: + log.info( + 'Port %d unexpectedly in use, last owning pid %d.', + candidate.port, candidate.pid) + + log.info('All ports in use.') + self.ports_checked_for_last_request = check_count + return 0 + + def add_port_to_free_pool(self, port): + """Add a new port to the free pool for allocation.""" + if port < 1 or port > 65535: + raise ValueError( + 'Port must be in the [1, 65535] range, not %d.' % port) + port_info = _PortInfo(port=port) + self._port_queue.append(port_info) + + +class _PortServerRequestHandler(object): + """A class to handle port allocation and status requests. + + Allocates ports to process ids via the dead simple port server protocol + when the handle_port_request asyncio.coroutine handler has been registered. + Statistics can be logged using the dump_stats method. + """ + + def __init__(self, ports_to_serve): + """Initialize a new port server. + + Args: + ports_to_serve: A sequence of unique port numbers to test and offer + up to clients. + """ + self._port_pool = _PortPool() + self._total_allocations = 0 + self._denied_allocations = 0 + self._client_request_errors = 0 + for port in ports_to_serve: + self._port_pool.add_port_to_free_pool(port) + + async def handle_port_request(self, reader, writer): + client_data = await reader.read(100) + self._handle_port_request(client_data, writer) + writer.close() + + def _handle_port_request(self, client_data, writer): + """Given a port request body, parse it and respond appropriately. + + Args: + client_data: The request bytes from the client. + writer: The asyncio Writer for the response to be written to. + """ + try: + if len(client_data) > 20: + raise ValueError('More than 20 characters in "pid".') + pid = int(client_data) + except ValueError as error: + self._client_request_errors += 1 + log.warning('Could not parse request: %s', error) + return + + log.info('Request on behalf of pid %d.', pid) + log.info('cmdline: %s', _get_process_command_line(pid)) + + if not _should_allocate_port(pid): + self._denied_allocations += 1 + return + + port = self._port_pool.get_port_for_process(pid) + if port > 0: + self._total_allocations += 1 + writer.write('{:d}\n'.format(port).encode('utf-8')) + log.debug('Allocated port %d to pid %d', port, pid) + else: + self._denied_allocations += 1 + + def dump_stats(self): + """Logs statistics of our operation.""" + log.info('Dumping statistics:') + stats = [] + stats.append( + 'client-request-errors {}'.format(self._client_request_errors)) + stats.append('denied-allocations {}'.format(self._denied_allocations)) + stats.append('num-ports-managed {}'.format(self._port_pool.num_ports())) + stats.append('num-ports-checked-for-last-request {}'.format( + self._port_pool.ports_checked_for_last_request)) + stats.append('total-allocations {}'.format(self._total_allocations)) + for stat in stats: + log.info(stat) + + +def _parse_command_line(): + """Configure and parse our command line flags.""" + parser = argparse.ArgumentParser() + parser.add_argument( + '--portserver_static_pool', + type=str, + default='15000-24999', + help='Comma separated N-P Range(s) of ports to manage (inclusive).') + parser.add_argument( + '--portserver_address', + '--portserver_unix_socket_address', # Alias to be backward compatible + type=str, + default='@unittest-portserver', + help='Address of AF_UNIX socket on which to listen on Unix (first @ is ' + 'a NUL) or the name of the pipe on Windows (first @ is the ' + r'\\.\pipe\ prefix).') + parser.add_argument('--verbose', + action='store_true', + default=False, + help='Enable verbose messages.') + parser.add_argument('--debug', + action='store_true', + default=False, + help='Enable full debug messages.') + return parser.parse_args(sys.argv[1:]) + + +def _parse_port_ranges(pool_str): + """Given a 'N-P,X-Y' description of port ranges, return a set of ints.""" + ports = set() + for range_str in pool_str.split(','): + try: + a, b = range_str.split('-', 1) + start, end = int(a), int(b) + except ValueError: + log.error('Ignoring unparsable port range %r.', range_str) + continue + if start < 1 or end > 65535: + log.error('Ignoring out of bounds port range %r.', range_str) + continue + ports.update(set(range(start, end + 1))) + return ports + + +def _configure_logging(verbose=False, debug=False): + """Configure the log global, message format, and verbosity settings.""" + overall_level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + format=('{levelname[0]}{asctime}.{msecs:03.0f} {thread} ' + '{filename}:{lineno}] {message}'), + datefmt='%m%d %H:%M:%S', + style='{', + level=overall_level) + global log + log = logging.getLogger('portserver') + # The verbosity controls our loggers logging level, not the global + # one above. This avoids debug messages from libraries such as asyncio. + log.setLevel(logging.DEBUG if verbose else overall_level) + + +def main(): + config = _parse_command_line() + if config.debug: + # Equivalent of PYTHONASYNCIODEBUG=1 in 3.4; pylint: disable=protected-access + asyncio.tasks._DEBUG = True + _configure_logging(verbose=config.verbose, debug=config.debug) + ports_to_serve = _parse_port_ranges(config.portserver_static_pool) + if not ports_to_serve: + log.error('No ports. Invalid port ranges in --portserver_static_pool?') + sys.exit(1) + + request_handler = _PortServerRequestHandler(ports_to_serve) + + if sys.platform == 'win32': + asyncio.set_event_loop(asyncio.ProactorEventLoop()) + + event_loop = asyncio.get_event_loop() + + if sys.platform == 'win32': + # On Windows, we need to periodically pause the loop to allow the user + # to send a break signal (e.g. ctrl+c) + def listen_for_signal(): + event_loop.call_later(0.5, listen_for_signal) + + event_loop.call_later(0.5, listen_for_signal) + + coro = _start_windows_server( + request_handler.handle_port_request, + path=config.portserver_address.replace('@', '\\\\.\\pipe\\', 1)) + else: + event_loop.add_signal_handler( + signal.SIGUSR1, request_handler.dump_stats) # pylint: disable=no-member + + old_py_loop = {'loop': event_loop} if sys.version_info < (3, 10) else {} + coro = asyncio.start_unix_server( + request_handler.handle_port_request, + path=config.portserver_address.replace('@', '\0', 1), + **old_py_loop) + + server_address = config.portserver_address + + server = event_loop.run_until_complete(coro) + log.info('Serving on %s', server_address) + try: + event_loop.run_forever() + except KeyboardInterrupt: + log.info('Stopping due to ^C.') + + server.close() + + if sys.platform != 'win32': + # PipeServer doesn't have a wait_closed() function + event_loop.run_until_complete(server.wait_closed()) + event_loop.remove_signal_handler(signal.SIGUSR1) # pylint: disable=no-member + + event_loop.close() + request_handler.dump_stats() + log.info('Goodbye.') + + +if __name__ == '__main__': + main() diff --git a/src/tests/portpicker_test.py b/src/tests/portpicker_test.py new file mode 100644 index 0000000..c2925db --- /dev/null +++ b/src/tests/portpicker_test.py @@ -0,0 +1,390 @@ +#!/usr/bin/python +# +# Copyright 2007 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Unittests for the portpicker module.""" + +from __future__ import print_function +import errno +import os +import random +import socket +import sys +import unittest +from contextlib import ExitStack + +if sys.platform == 'win32': + import _winapi +else: + _winapi = None + +try: + # pylint: disable=no-name-in-module + from unittest import mock # Python >= 3.3. +except ImportError: + import mock # https://pypi.python.org/pypi/mock + +import portpicker + + +class PickUnusedPortTest(unittest.TestCase): + def IsUnusedTCPPort(self, port): + return self._bind(port, socket.SOCK_STREAM, socket.IPPROTO_TCP) + + def IsUnusedUDPPort(self, port): + return self._bind(port, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + + def setUp(self): + # So we can Bind even if portpicker.bind is stubbed out. + self._bind = portpicker.bind + portpicker._owned_ports.clear() + portpicker._free_ports.clear() + portpicker._random_ports.clear() + + def testPickUnusedPortActuallyWorks(self): + """This test can be flaky.""" + for _ in range(10): + port = portpicker.pick_unused_port() + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + + @unittest.skipIf('PORTSERVER_ADDRESS' not in os.environ, + 'no port server to test against') + def testPickUnusedCanSuccessfullyUsePortServer(self): + + with mock.patch.object(portpicker, '_pick_unused_port_without_server'): + portpicker._pick_unused_port_without_server.side_effect = ( + Exception('eek!') + ) + + # Since _PickUnusedPortWithoutServer() raises an exception, if we + # can successfully obtain a port, the portserver must be working. + port = portpicker.pick_unused_port() + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + + @unittest.skipIf('PORTSERVER_ADDRESS' not in os.environ, + 'no port server to test against') + def testPickUnusedCanSuccessfullyUsePortServerAddressKwarg(self): + + with mock.patch.object(portpicker, '_pick_unused_port_without_server'): + portpicker._pick_unused_port_without_server.side_effect = ( + Exception('eek!') + ) + + # Since _PickUnusedPortWithoutServer() raises an exception, and + # we've temporarily removed PORTSERVER_ADDRESS from os.environ, if + # we can successfully obtain a port, the portserver must be working. + addr = os.environ.pop('PORTSERVER_ADDRESS') + try: + port = portpicker.pick_unused_port(portserver_address=addr) + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + finally: + os.environ['PORTSERVER_ADDRESS'] = addr + + @unittest.skipIf('PORTSERVER_ADDRESS' not in os.environ, + 'no port server to test against') + def testGetPortFromPortServer(self): + """Exercise the get_port_from_port_server() helper function.""" + for _ in range(10): + port = portpicker.get_port_from_port_server( + os.environ['PORTSERVER_ADDRESS']) + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + + def testSendsPidToPortServer(self): + with ExitStack() as stack: + if _winapi: + create_file_mock = mock.Mock() + create_file_mock.return_value = 0 + read_file_mock = mock.Mock() + write_file_mock = mock.Mock() + read_file_mock.return_value = (b'42768\n', 0) + stack.enter_context( + mock.patch('_winapi.CreateFile', new=create_file_mock)) + stack.enter_context( + mock.patch('_winapi.WriteFile', new=write_file_mock)) + stack.enter_context( + mock.patch('_winapi.ReadFile', new=read_file_mock)) + port = portpicker.get_port_from_port_server( + 'portserver', pid=1234) + write_file_mock.assert_called_once_with(0, b'1234\n') + else: + server = mock.Mock() + server.recv.return_value = b'42768\n' + stack.enter_context( + mock.patch.object(socket, 'socket', return_value=server)) + port = portpicker.get_port_from_port_server( + 'portserver', pid=1234) + server.sendall.assert_called_once_with(b'1234\n') + + self.assertEqual(port, 42768) + + def testPidDefaultsToOwnPid(self): + with ExitStack() as stack: + stack.enter_context( + mock.patch.object(os, 'getpid', return_value=9876)) + + if _winapi: + create_file_mock = mock.Mock() + create_file_mock.return_value = 0 + read_file_mock = mock.Mock() + write_file_mock = mock.Mock() + read_file_mock.return_value = (b'52768\n', 0) + stack.enter_context( + mock.patch('_winapi.CreateFile', new=create_file_mock)) + stack.enter_context( + mock.patch('_winapi.WriteFile', new=write_file_mock)) + stack.enter_context( + mock.patch('_winapi.ReadFile', new=read_file_mock)) + port = portpicker.get_port_from_port_server('portserver') + write_file_mock.assert_called_once_with(0, b'9876\n') + else: + server = mock.Mock() + server.recv.return_value = b'52768\n' + stack.enter_context( + mock.patch.object(socket, 'socket', return_value=server)) + port = portpicker.get_port_from_port_server('portserver') + server.sendall.assert_called_once_with(b'9876\n') + + self.assertEqual(port, 52768) + + @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': 'portserver'}) + def testReusesPortServerPorts(self): + with ExitStack() as stack: + if _winapi: + read_file_mock = mock.Mock() + read_file_mock.side_effect = [ + (b'12345\n', 0), + (b'23456\n', 0), + (b'34567\n', 0), + ] + stack.enter_context(mock.patch('_winapi.CreateFile')) + stack.enter_context(mock.patch('_winapi.WriteFile')) + stack.enter_context( + mock.patch('_winapi.ReadFile', new=read_file_mock)) + else: + server = mock.Mock() + server.recv.side_effect = [b'12345\n', b'23456\n', b'34567\n'] + stack.enter_context( + mock.patch.object(socket, 'socket', return_value=server)) + + self.assertEqual(portpicker.pick_unused_port(), 12345) + self.assertEqual(portpicker.pick_unused_port(), 23456) + portpicker.return_port(12345) + self.assertEqual(portpicker.pick_unused_port(), 12345) + + @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': ''}) + def testDoesntReuseRandomPorts(self): + ports = set() + for _ in range(10): + try: + port = portpicker.pick_unused_port() + except portpicker.NoFreePortFoundError: + # This sometimes happens when not using portserver. Just + # skip to the next attempt. + continue + ports.add(port) + portpicker.return_port(port) + self.assertGreater(len(ports), 5) # Allow some random reuse. + + def testReturnsReservedPorts(self): + with mock.patch.object(portpicker, '_pick_unused_port_without_server'): + portpicker._pick_unused_port_without_server.side_effect = ( + Exception('eek!')) + # Arbitrary port. In practice you should get this from somewhere + # that assigns ports. + reserved_port = 28465 + portpicker.add_reserved_port(reserved_port) + ports = set() + for _ in range(10): + port = portpicker.pick_unused_port() + ports.add(port) + portpicker.return_port(port) + self.assertEqual(len(ports), 1) + self.assertEqual(ports.pop(), reserved_port) + + @mock.patch.dict(os.environ,{'PORTSERVER_ADDRESS': ''}) + def testFallsBackToRandomAfterRunningOutOfReservedPorts(self): + # Arbitrary port. In practice you should get this from somewhere + # that assigns ports. + reserved_port = 23456 + portpicker.add_reserved_port(reserved_port) + self.assertEqual(portpicker.pick_unused_port(), reserved_port) + self.assertNotEqual(portpicker.pick_unused_port(), reserved_port) + + def testRandomlyChosenPorts(self): + # Unless this box is under an overwhelming socket load, this test + # will heavily exercise the "pick a port randomly" part of the + # port picking code, but may never hit the "OS assigns a port" + # code. + ports = 0 + for _ in range(100): + try: + port = portpicker._pick_unused_port_without_server() + except portpicker.NoFreePortFoundError: + # Without the portserver, pick_unused_port can sometimes fail + # to find a free port. Check that it passes most of the time. + continue + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + ports += 1 + # Getting a port shouldn't have failed very often, even on machines + # with a heavy socket load. + self.assertGreater(ports, 95) + + def testOSAssignedPorts(self): + self.last_assigned_port = None + + def error_for_explicit_ports(port, socket_type, socket_proto): + # Only successfully return a port if an OS-assigned port is + # requested, or if we're checking that the last OS-assigned port + # is unused on the other protocol. + if port == 0 or port == self.last_assigned_port: + self.last_assigned_port = self._bind(port, socket_type, + socket_proto) + return self.last_assigned_port + else: + return None + + with mock.patch.object(portpicker, 'bind', error_for_explicit_ports): + # Without server, this can be little flaky, so check that it + # passes most of the time. + ports = 0 + for _ in range(100): + try: + port = portpicker._pick_unused_port_without_server() + except portpicker.NoFreePortFoundError: + continue + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + ports += 1 + self.assertGreater(ports, 70) + + def pickUnusedPortWithoutServer(self): + # Try a few times to pick a port, to avoid flakiness and to make sure + # the code path we want was exercised. + for _ in range(5): + try: + port = portpicker._pick_unused_port_without_server() + except portpicker.NoFreePortFoundError: + continue + else: + self.assertTrue(self.IsUnusedTCPPort(port)) + self.assertTrue(self.IsUnusedUDPPort(port)) + return + self.fail("Failed to find a free port") + + def testPickPortsWithoutServer(self): + # Test the first part of _pick_unused_port_without_server, which + # tries a few random ports and checks is_port_free. + self.pickUnusedPortWithoutServer() + + # Now test the second part, the fallback from above, which asks the + # OS for a port. + def mock_port_free(port): + return False + + with mock.patch.object(portpicker, 'is_port_free', mock_port_free): + self.pickUnusedPortWithoutServer() + + def checkIsPortFree(self): + """This might be flaky unless this test is run with a portserver.""" + # The port should be free initially. + port = portpicker.pick_unused_port() + self.assertTrue(portpicker.is_port_free(port)) + + cases = [ + (socket.AF_INET, socket.SOCK_STREAM, None), + (socket.AF_INET6, socket.SOCK_STREAM, 1), + (socket.AF_INET, socket.SOCK_DGRAM, None), + (socket.AF_INET6, socket.SOCK_DGRAM, 1), + ] + + # Using v6only=0 on Windows doesn't result in collisions + if not _winapi: + cases.extend([ + (socket.AF_INET6, socket.SOCK_STREAM, 0), + (socket.AF_INET6, socket.SOCK_DGRAM, 0), + ]) + + for (sock_family, sock_type, v6only) in cases: + # Occupy the port on a subset of possible protocols. + try: + sock = socket.socket(sock_family, sock_type, 0) + except socket.error: + print('Kernel does not support sock_family=%d' % sock_family, + file=sys.stderr) + # Skip this case, since we cannot occupy a port. + continue + + if not hasattr(socket, 'IPPROTO_IPV6'): + v6only = None + + if v6only is not None: + try: + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, + v6only) + except socket.error: + print('Kernel does not support IPV6_V6ONLY=%d' % v6only, + file=sys.stderr) + # Don't care; just proceed with the default. + + # Socket may have been taken in the mean time, so catch the + # socket.error with errno set to EADDRINUSE and skip this + # attempt. + try: + sock.bind(('', port)) + except socket.error as e: + if e.errno == errno.EADDRINUSE: + raise portpicker.NoFreePortFoundError + raise + + # The port should be busy. + self.assertFalse(portpicker.is_port_free(port)) + sock.close() + + # Now it's free again. + self.assertTrue(portpicker.is_port_free(port)) + + def testIsPortFree(self): + # This can be quite flaky on a busy host, so try a few times. + for _ in range(10): + try: + self.checkIsPortFree() + except portpicker.NoFreePortFoundError: + pass + else: + return + self.fail("checkPortIsFree failed every time.") + + def testIsPortFreeException(self): + port = portpicker.pick_unused_port() + with mock.patch.object(socket, 'socket') as mock_sock: + mock_sock.side_effect = socket.error('fake socket error', 0) + self.assertFalse(portpicker.is_port_free(port)) + + def testThatLegacyCapWordsAPIsExist(self): + """The original APIs were CapWords style, 1.1 added PEP8 names.""" + self.assertEqual(portpicker.bind, portpicker.Bind) + self.assertEqual(portpicker.is_port_free, portpicker.IsPortFree) + self.assertEqual(portpicker.pick_unused_port, portpicker.PickUnusedPort) + self.assertEqual(portpicker.get_port_from_port_server, + portpicker.GetPortFromPortServer) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/tests/portserver_test.py b/src/tests/portserver_test.py new file mode 100644 index 0000000..b7de094 --- /dev/null +++ b/src/tests/portserver_test.py @@ -0,0 +1,370 @@ +#!/usr/bin/python3 +# +# Copyright 2015 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Tests for the example portserver.""" + +import asyncio +import os +import signal +import socket +import subprocess +import sys +import time +import unittest +from unittest import mock +from multiprocessing import Process + +import portpicker + +# On Windows, portserver.py is located in the "Scripts" folder, which isn't +# added to the import path by default +if sys.platform == 'win32': + sys.path.append(os.path.join(os.path.split(sys.executable)[0])) + +import portserver + + +def setUpModule(): + portserver._configure_logging(verbose=True) + +def exit_immediately(): + os._exit(0) + +class PortserverFunctionsTest(unittest.TestCase): + + @classmethod + def setUp(cls): + cls.port = portpicker.PickUnusedPort() + + def test_get_process_command_line(self): + portserver._get_process_command_line(os.getpid()) + + def test_get_process_start_time(self): + self.assertGreater(portserver._get_process_start_time(os.getpid()), 0) + + def test_is_port_free(self): + """This might be flaky unless this test is run with a portserver.""" + # The port should be free initially. + self.assertTrue(portserver._is_port_free(self.port)) + + cases = [ + (socket.AF_INET, socket.SOCK_STREAM, None), + (socket.AF_INET6, socket.SOCK_STREAM, 1), + (socket.AF_INET, socket.SOCK_DGRAM, None), + (socket.AF_INET6, socket.SOCK_DGRAM, 1), + ] + + # Using v6only=0 on Windows doesn't result in collisions + if sys.platform != 'win32': + cases.extend([ + (socket.AF_INET6, socket.SOCK_STREAM, 0), + (socket.AF_INET6, socket.SOCK_DGRAM, 0), + ]) + + for (sock_family, sock_type, v6only) in cases: + # Occupy the port on a subset of possible protocols. + try: + sock = socket.socket(sock_family, sock_type, 0) + except socket.error: + print('Kernel does not support sock_family=%d' % sock_family, + file=sys.stderr) + # Skip this case, since we cannot occupy a port. + continue + + if not hasattr(socket, 'IPPROTO_IPV6'): + v6only = None + + if v6only is not None: + try: + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, + v6only) + except socket.error: + print('Kernel does not support IPV6_V6ONLY=%d' % v6only, + file=sys.stderr) + # Don't care; just proceed with the default. + sock.bind(('', self.port)) + + # The port should be busy. + self.assertFalse(portserver._is_port_free(self.port)) + sock.close() + + # Now it's free again. + self.assertTrue(portserver._is_port_free(self.port)) + + def test_is_port_free_exception(self): + with mock.patch.object(socket, 'socket') as mock_sock: + mock_sock.side_effect = socket.error('fake socket error', 0) + self.assertFalse(portserver._is_port_free(self.port)) + + def test_should_allocate_port(self): + self.assertFalse(portserver._should_allocate_port(0)) + self.assertFalse(portserver._should_allocate_port(1)) + self.assertTrue(portserver._should_allocate_port, os.getpid()) + + p = Process(target=exit_immediately) + p.start() + child_pid = p.pid + p.join() + + # This test assumes that after waitpid returns the kernel has finished + # cleaning the process. We also assume that the kernel will not reuse + # the former child's pid before our next call checks for its existence. + # Likely assumptions, but not guaranteed. + self.assertFalse(portserver._should_allocate_port(child_pid)) + + def test_parse_command_line(self): + with mock.patch.object( + sys, 'argv', ['program_name', '--verbose', + '--portserver_static_pool=1-1,3-8', + '--portserver_unix_socket_address=@hello-test']): + portserver._parse_command_line() + + def test_parse_port_ranges(self): + self.assertFalse(portserver._parse_port_ranges('')) + self.assertCountEqual(portserver._parse_port_ranges('1-1'), {1}) + self.assertCountEqual(portserver._parse_port_ranges('1-1,3-8,375-378'), + {1, 3, 4, 5, 6, 7, 8, 375, 376, 377, 378}) + # Unparsable parts are logged but ignored. + self.assertEqual({1, 2}, + portserver._parse_port_ranges('1-2,not,numbers')) + self.assertEqual(set(), portserver._parse_port_ranges('8080-8081x')) + # Port ranges that go out of bounds are logged but ignored. + self.assertEqual(set(), portserver._parse_port_ranges('0-1138')) + self.assertEqual(set(range(19, 84 + 1)), + portserver._parse_port_ranges('1138-65536,19-84')) + + def test_configure_logging(self): + """Just code coverage really.""" + portserver._configure_logging(False) + portserver._configure_logging(True) + + + _test_socket_addr = f'@TST-{os.getpid()}' + + @mock.patch.object( + sys, 'argv', ['PortserverFunctionsTest.test_main', + f'--portserver_unix_socket_address={_test_socket_addr}'] + ) + @mock.patch.object(portserver, '_parse_port_ranges') + def test_main_no_ports(self, *unused_mocks): + portserver._parse_port_ranges.return_value = set() + with self.assertRaises(SystemExit): + portserver.main() + + @unittest.skipUnless(sys.executable, 'Requires a stand alone interpreter') + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'AF_UNIX required') + def test_portserver_binary(self): + """Launch python portserver.py and test it.""" + # Blindly assuming tree layout is src/tests/portserver_test.py + # with src/portserver.py. + portserver_py = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'portserver.py') + anon_addr = self._test_socket_addr.replace('@', '\0') + + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + with self.assertRaises( + ConnectionRefusedError, + msg=f'{self._test_socket_addr} should not listen yet.'): + conn.connect(anon_addr) + conn.close() + + server = subprocess.Popen( + [sys.executable, portserver_py, + f'--portserver_unix_socket_address={self._test_socket_addr}'], + stderr=subprocess.PIPE, + ) + try: + # Wait a few seconds for the server to start listening. + start_time = time.monotonic() + while True: + time.sleep(0.05) + try: + conn.connect(anon_addr) + conn.close() + except ConnectionRefusedError: + delta = time.monotonic() - start_time + if delta < 4: + continue + else: + server.kill() + self.fail('Failed to connect to portserver ' + f'{self._test_socket_addr} within ' + f'{delta} seconds. STDERR:\n' + + server.stderr.read().decode('utf-8')) + else: + break + + ports = set() + port = portpicker.get_port_from_port_server( + portserver_address=self._test_socket_addr) + ports.add(port) + port = portpicker.get_port_from_port_server( + portserver_address=self._test_socket_addr) + ports.add(port) + + with subprocess.Popen('exit 0', shell=True) as quick_process: + quick_process.wait() + # This process doesn't exist so it should be a denied alloc. + # We use the pid from the above quick_process under the assumption + # that most OSes try to avoid rapid pid recycling. + denied_port = portpicker.get_port_from_port_server( + portserver_address=self._test_socket_addr, + pid=quick_process.pid) # A now unused pid. + self.assertIsNone(denied_port) + + self.assertEqual(len(ports), 2, msg=ports) + + # Check statistics from portserver + server.send_signal(signal.SIGUSR1) + # TODO implement an I/O timeout + for line in server.stderr: + if b'denied-allocations ' in line: + denied_allocations = int( + line.split(b'denied-allocations ', 2)[1]) + self.assertEqual(1, denied_allocations, msg=line) + elif b'total-allocations ' in line: + total_allocations = int( + line.split(b'total-allocations ', 2)[1]) + self.assertEqual(2, total_allocations, msg=line) + break + + rejected_port = portpicker.get_port_from_port_server( + portserver_address=self._test_socket_addr, + pid=99999999999999999999999999999999999) # Out of range. + self.assertIsNone(rejected_port) + + # Done. shutdown gracefully. + server.send_signal(signal.SIGINT) + server.communicate(timeout=2) + finally: + server.kill() + server.wait() + + +class PortPoolTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.port = portpicker.PickUnusedPort() + + def setUp(self): + self.pool = portserver._PortPool() + + def test_initialization(self): + self.assertEqual(0, self.pool.num_ports()) + self.pool.add_port_to_free_pool(self.port) + self.assertEqual(1, self.pool.num_ports()) + self.pool.add_port_to_free_pool(1138) + self.assertEqual(2, self.pool.num_ports()) + self.assertRaises(ValueError, self.pool.add_port_to_free_pool, 0) + self.assertRaises(ValueError, self.pool.add_port_to_free_pool, 65536) + + @mock.patch.object(portserver, '_is_port_free') + def test_get_port_for_process_ok(self, mock_is_port_free): + self.pool.add_port_to_free_pool(self.port) + mock_is_port_free.return_value = True + self.assertEqual(self.port, self.pool.get_port_for_process(os.getpid())) + self.assertEqual(1, self.pool.ports_checked_for_last_request) + + @mock.patch.object(portserver, '_is_port_free') + def test_get_port_for_process_none_left(self, mock_is_port_free): + self.pool.add_port_to_free_pool(self.port) + self.pool.add_port_to_free_pool(22) + mock_is_port_free.return_value = False + self.assertEqual(2, self.pool.num_ports()) + self.assertEqual(0, self.pool.get_port_for_process(os.getpid())) + self.assertEqual(2, self.pool.num_ports()) + self.assertEqual(2, self.pool.ports_checked_for_last_request) + + @mock.patch.object(portserver, '_is_port_free') + @mock.patch.object(os, 'getpid') + def test_get_port_for_process_pid_eq_port(self, mock_getpid, mock_is_port_free): + self.pool.add_port_to_free_pool(12345) + self.pool.add_port_to_free_pool(12344) + mock_is_port_free.side_effect = lambda port: port == os.getpid() + mock_getpid.return_value = 12345 + self.assertEqual(2, self.pool.num_ports()) + self.assertEqual(12345, self.pool.get_port_for_process(os.getpid())) + self.assertEqual(2, self.pool.ports_checked_for_last_request) + + @mock.patch.object(portserver, '_is_port_free') + @mock.patch.object(os, 'getpid') + def test_get_port_for_process_pid_ne_port(self, mock_getpid, mock_is_port_free): + self.pool.add_port_to_free_pool(12344) + self.pool.add_port_to_free_pool(12345) + mock_is_port_free.side_effect = lambda port: port != os.getpid() + mock_getpid.return_value = 12345 + self.assertEqual(2, self.pool.num_ports()) + self.assertEqual(12344, self.pool.get_port_for_process(os.getpid())) + self.assertEqual(2, self.pool.ports_checked_for_last_request) + + +@mock.patch.object(portserver, '_get_process_command_line') +@mock.patch.object(portserver, '_should_allocate_port') +@mock.patch.object(portserver._PortPool, 'get_port_for_process') +class PortServerRequestHandlerTest(unittest.TestCase): + def setUp(self): + portserver._configure_logging(verbose=True) + self.rh = portserver._PortServerRequestHandler([23, 42, 54]) + + def test_stats_reporting(self, *unused_mocks): + with mock.patch.object(portserver, 'log') as mock_logger: + self.rh.dump_stats() + mock_logger.info.assert_called_with('total-allocations 0') + + def test_handle_port_request_bad_data(self, *unused_mocks): + self._test_bad_data_from_client(b'') + self._test_bad_data_from_client(b'\n') + self._test_bad_data_from_client(b'99Z\n') + self._test_bad_data_from_client(b'99 8\n') + self.assertEqual([], portserver._get_process_command_line.mock_calls) + + def _test_bad_data_from_client(self, data): + mock_writer = mock.Mock(asyncio.StreamWriter) + self.rh._handle_port_request(data, mock_writer) + self.assertFalse(portserver._should_allocate_port.mock_calls) + + def test_handle_port_request_denied_allocation(self, *unused_mocks): + portserver._should_allocate_port.return_value = False + self.assertEqual(0, self.rh._denied_allocations) + mock_writer = mock.Mock(asyncio.StreamWriter) + self.rh._handle_port_request(b'5\n', mock_writer) + self.assertEqual(1, self.rh._denied_allocations) + + def test_handle_port_request_bad_port_returned(self, *unused_mocks): + portserver._should_allocate_port.return_value = True + self.rh._port_pool.get_port_for_process.return_value = 0 + mock_writer = mock.Mock(asyncio.StreamWriter) + self.rh._handle_port_request(b'6\n', mock_writer) + self.rh._port_pool.get_port_for_process.assert_called_once_with(6) + self.assertEqual(1, self.rh._denied_allocations) + + def test_handle_port_request_success(self, *unused_mocks): + portserver._should_allocate_port.return_value = True + self.rh._port_pool.get_port_for_process.return_value = 999 + mock_writer = mock.Mock(asyncio.StreamWriter) + self.assertEqual(0, self.rh._total_allocations) + self.rh._handle_port_request(b'8', mock_writer) + portserver._should_allocate_port.assert_called_once_with(8) + self.rh._port_pool.get_port_for_process.assert_called_once_with(8) + self.assertEqual(1, self.rh._total_allocations) + self.assertEqual(0, self.rh._denied_allocations) + mock_writer.write.assert_called_once_with(b'999\n') + + +if __name__ == '__main__': + unittest.main() @@ -0,0 +1,12 @@ +#!/bin/sh -ex + +unset PYTHONPATH +python3 -m venv build/venv +. build/venv/bin/activate + +pip install --upgrade pip +pip install tox +# We should really do this differently, test from a `pip install .` so that +# testing relies on the setup.cfg install_requires instead of listing it here. +pip install psutil +tox -e "py3$(python -c 'import sys; print(sys.version_info.minor)')" |