Upgrade python/httplib2 to v0.18.1 am: f6b4dcdd5aandroid-r-beta-3android-r-beta-2

Change-Id: I23d7334a933cad86e271dfaaa038240d149458c4

12 files changed, 114 insertions, 19 deletions

diff --git a/.travis.yml b/.travis.yml
index 395c507..ed1aa72 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,7 @@ cache:
 
 env:
   global:
-    - pip_install_common='pip>=9.0 setuptools>=36.2 wheel>=0.30'
+    - pip_install_common='pip>=9.0 setuptools>=43.0 wheel>=0.30'
 python:
   - 2.7
   - 3.5
diff --git a/CHANGELOG b/CHANGELOG
index ba20b24..2db1cc0 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,26 @@
+0.18.1
+
+  explicit build-backend workaround for pip build isolation bug
+  "AttributeError: 'module' object has no attribute '__legacy__'" on pip install
+  https://github.com/httplib2/httplib2/issues/169
+
+0.18.0
+
+  IMPORTANT security vulnerability CWE-93 CRLF injection
+  Force %xx quote of space, CR, LF characters in uri.
+  Special thanks to Recar https://github.com/Ciyfly for discrete notification.
+  https://cwe.mitre.org/data/definitions/93.html
+
+0.17.4
+
+  Ship test suite in source dist
+  https://github.com/httplib2/httplib2/pull/168
+
+0.17.3
+
+  IronPython2.7: relative import iri2uri fixes ImportError
+  https://github.com/httplib2/httplib2/pull/163
+
 0.17.2
 
   python3 + debug + IPv6 disabled: https raised
diff --git a/MANIFEST.in b/MANIFEST.in
index 12c4cc7..412def6 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,11 @@
 recursive-include python2 *.py *.txt
 recursive-include python3 *.py *.txt
+graft test
+graft tests
+include *.md
+include CHANGELOG
+include LICENSE
 include python2/httplib2/test/*.txt
 include requirements*.txt
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/METADATA b/METADATA
index f393882..ac99305 100644
--- a/METADATA
+++ b/METADATA
@@ -9,11 +9,11 @@ third_party {
     type: GIT
     value: "https://github.com/httplib2/httplib2.git"
   }
-  version: "v0.17.2"
+  version: "v0.18.1"
   license_type: NOTICE
   last_upgrade_date {
     year: 2020
-    month: 4
-    day: 13
+    month: 5
+    day: 20
   }
 }
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..5eb3903
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,19 @@
+# Security Policy
+
+## Supported Versions
+
+master branch and latest release get priority support. You should expect all known problems fixed in master.
+
+All other released versions receive security updates per request.
+If you use some old version and can not upgrade for any or no reason, ask for security update release, most likely you will get it.
+
+## Reporting a Vulnerability
+
+Contact current maintainers. At 2020-05: temotor@gmail.com or https://t.me/temotor
+If that doesn't work, open Github issue just asking for private communication channel.
+
+This is volunteer maintained project, all issues are processed on best effort basis, no deadlines promised. Of course, security vulnerabilities get priority over regular issues.
+
+You can expect fame in history or maybe you prefer anonymity - say what you prefer.
+
+Thank you for responsible handling of security problems. Your attention and effort are appreciated.
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..5f7cbbd
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,9 @@
+[build-system]
+requires = ["setuptools >= 40.8.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 121
+
+[tool.check-manifest]
+ignore = [".travis.yml", "script/*", "*.tex"]
diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py
index 9057d1f..f35ba48 100644
--- a/python2/httplib2/__init__.py
+++ b/python2/httplib2/__init__.py
@@ -19,7 +19,7 @@ __contributors__ = [
     "Alex Yu",
 ]
 __license__ = "MIT"
-__version__ = '0.17.2'
+__version__ = "0.18.1"
 
 import base64
 import calendar
@@ -129,7 +129,7 @@ if ssl is None:
     _ssl_wrap_socket = _ssl_wrap_socket_unsupported
 
 if sys.version_info >= (2, 3):
-    from iri2uri import iri2uri
+    from .iri2uri import iri2uri
 else:
 
     def iri2uri(uri):
@@ -1985,6 +1985,9 @@ class Http(object):
                 headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__
 
             uri = iri2uri(uri)
+            # Prevent CWE-75 space injection to manipulate request via part of uri.
+            # Prevent CWE-93 CRLF injection to modify headers via part of uri.
+            uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A")
 
             (scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
 
diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py
index 135c6f6..cf2db60 100644
--- a/python3/httplib2/__init__.py
+++ b/python3/httplib2/__init__.py
@@ -15,7 +15,7 @@ __contributors__ = [
     "Alex Yu",
 ]
 __license__ = "MIT"
-__version__ = '0.17.2'
+__version__ = "0.18.1"
 
 import base64
 import calendar
@@ -1790,6 +1790,9 @@ a string that contains the response entity body.
                 headers["user-agent"] = "Python-httplib2/%s (gzip)" % __version__
 
             uri = iri2uri(uri)
+            # Prevent CWE-75 space injection to manipulate request via part of uri.
+            # Prevent CWE-93 CRLF injection to modify headers via part of uri.
+            uri = uri.replace(" ", "%20").replace("\r", "%0D").replace("\n", "%0A")
 
             (scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
 
diff --git a/script/release b/script/release
index 0f98e3e..a2ff80d 100755
--- a/script/release
+++ b/script/release
@@ -45,9 +45,9 @@ auto_prepare_release() {
 		last_tag=$(git tag --sort=-version:refname |head -n1)
 		last_tag=${last_tag##v}
 		version_replace="${last_tag}.post$(date -u +%y%m%d%H%M)"
-		update_version "setup.py" "s/VERSION =.+/VERSION = '$version_replace'/"
-		update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_replace'/"
-		update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_replace'/"
+		update_version "setup.py" "s/VERSION =.+/VERSION = \"$version_replace\"/"
+		update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_replace\"/"
+		update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_replace\"/"
 		version_check "$version_replace"
 	fi
 }
@@ -90,10 +90,11 @@ interactive() {
 	local venv=./venv-release
 	if [[ ! -d "$venv" ]] ; then
 		virtualenv $venv
-		$venv/bin/pip install -U pip setuptools wheel twine
+		$venv/bin/pip install -U check-manifest pip 'setuptools>=43.0' wheel twine
 	fi
 	$venv/bin/python setup.py clean --all
 	$venv/bin/python setup.py sdist bdist_wheel
+	$venv/bin/check-manifest || echo "FIXME check-manifest" >&2
 
 	if confirm "Upload to PyPI? Use in special situation, normally CI (Travis) will upload to PyPI. [yN] " ; then
 		$venv/bin/twine upload dist/* || exit 1
@@ -132,9 +133,9 @@ bump_version() {
 	fi
 	echo "Next version:    '$version_next'" >&2
 
-	update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_next'/"
-	update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = '$version_next'/"
-	update_version "setup.py" "s/VERSION =.+/VERSION = '$version_next'/"
+	update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_next\"/"
+	update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_next\"/"
+	update_version "setup.py" "s/VERSION =.+/VERSION = \"$version_next\"/"
 
 	confirm "Confirm changes? [yN] " || exit 1
 }
@@ -142,8 +143,8 @@ bump_version() {
 update_version() {
 	local path="$1"
 	local sed_expr="$2"
-		# sed -E --in-place='' -e "s/VERSION =.+/VERSION = '$version_replace'/" setup.py
-		# sed -E --in-place='' -e "s/__version__ =.+/__version__ = '$version_replace'/" python2/httplib2/__init__.py python3/httplib2/__init__.py
+		# sed -E --in-place='' -e "s/VERSION =.+/VERSION = \"$version_replace\"/" setup.py
+		# sed -E --in-place='' -e "s/__version__ =.+/__version__ = \"$version_replace\"/" python2/httplib2/__init__.py python3/httplib2/__init__.py
 	echo "Updating file '$path'" >&2
 	if ! sed -E --in-place='' -e "$sed_expr" "$path" ; then
 		echo "sed error $?" >&2
@@ -209,7 +210,7 @@ assert_tree_clean() {
 
 version_check() {
 	local need=$1
-	local version_setup=$(fgrep 'VERSION =' setup.py |tr -d " '" |cut -d\= -f2)
+	local version_setup=$(fgrep 'VERSION =' setup.py |tr -d " '\"" |cut -d\= -f2)
 	local version_py2=$(cd python2 ; python2 -Es -c 'import httplib2;print(httplib2.__version__)')
 	local version_py3=$(cd python3 ; python3 -Es -c 'import httplib2;print(httplib2.__version__)')
 	if [[ "$version_setup" != "$need" ]] ; then
diff --git a/setup.py b/setup.py
index 9140845..b66d24e 100755
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,7 @@ import setuptools.command.test
 import sys
 
 pkgdir = {"": "python%s" % sys.version_info[0]}
-VERSION = '0.17.2'
+VERSION = "0.18.1"
 
 
 # `python setup.py test` uses existing Python environment, no virtualenv, no pip.
diff --git a/tests/__init__.py b/tests/__init__.py
index a15db9e..02a3ecf 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -75,7 +75,7 @@ class BufferedReader(object):
                 chunk = b""
             else:
                 chunk = self._sock.recv(8 << 10)
-            # print('!!! recv', chunk)
+            # print("!!! recv", chunk)
             if not chunk:
                 self._end = True
                 if untilend:
diff --git a/tests/test_http.py b/tests/test_http.py
index df99016..f61992c 100644
--- a/tests/test_http.py
+++ b/tests/test_http.py
@@ -703,3 +703,33 @@ def test_custom_redirect_codes():
         response, content = http.request(uri, "GET")
         assert response.status == 301
         assert response.previous is None
+
+
+def test_cwe93_inject_crlf():
+    # https://cwe.mitre.org/data/definitions/93.html
+    # GET /?q= HTTP/1.1      <- injected "HTTP/1.1" from attacker
+    # injected: attack
+    # ignore-http: HTTP/1.1  <- nominal "HTTP/1.1" from library
+    # Host: localhost:57285
+    http = httplib2.Http()
+    with tests.server_reflect() as uri:
+        danger_url = urllib.parse.urljoin(
+            uri, "?q= HTTP/1.1\r\ninjected: attack\r\nignore-http:"
+        )
+        response, content = http.request(danger_url, "GET")
+        assert response.status == 200
+        req = tests.HttpRequest.from_bytes(content)
+        assert req.headers.get("injected") is None
+
+
+def test_inject_space():
+    # Injecting space into request line is precursor to CWE-93 and possibly other injections
+    http = httplib2.Http()
+    with tests.server_reflect() as uri:
+        # "\r\nignore-http:" suffix is nuance for current server implementation
+        # please only pay attention to space after "?q="
+        danger_url = urllib.parse.urljoin(uri, "?q= HTTP/1.1\r\nignore-http:")
+        response, content = http.request(danger_url, "GET")
+        assert response.status == 200
+        req = tests.HttpRequest.from_bytes(content)
+        assert req.uri == "/?q=%20HTTP/1.1%0D%0Aignore-http:"