Compare commits
3 Commits
4366abba69
...
2dd664c4b4
| Author | SHA1 | Date | |
|---|---|---|---|
| 2dd664c4b4 | |||
| 1ec3ee7abf | |||
| dd808b1d68 |
@@ -1,2 +1,3 @@
|
||||
__pycache__/
|
||||
orchestrateur/db.sqlite-shm
|
||||
venv/
|
||||
+13
-10
@@ -20,9 +20,9 @@ db = client["microwave_network_db"]
|
||||
cooking_collection = db["cooking_parameters"]
|
||||
device_network_collection = db["device_network"]
|
||||
|
||||
# Ensure the photo storage directory exists when the app starts
|
||||
PHOTO_DIR = "storage/dishPhotos"
|
||||
os.makedirs(PHOTO_DIR, exist_ok=True)
|
||||
# Ensure the camera image storage directory exists when the app starts
|
||||
CAMERA_IMAGE_DIR = "storage/dishCameraImages"
|
||||
os.makedirs(CAMERA_IMAGE_DIR, exist_ok=True)
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Routes
|
||||
@@ -40,24 +40,24 @@ def cooking_params():
|
||||
if not data:
|
||||
return jsonify({"error": "Invalid or missing JSON payload"}), 400
|
||||
|
||||
# 1. Handle the Photo
|
||||
photo_b64 = data.get("photo")
|
||||
if photo_b64:
|
||||
# 1. Handle the Camera Image
|
||||
camera_image_b64 = data.get("camera_image")
|
||||
if camera_image_b64:
|
||||
# Generate a unique filename using UUID to avoid overwriting
|
||||
filename = f"dish_{uuid.uuid4().hex}.jpg"
|
||||
filepath = os.path.join(PHOTO_DIR, filename)
|
||||
filepath = os.path.join(CAMERA_IMAGE_DIR, filename)
|
||||
|
||||
try:
|
||||
# Decode the base64 string and save it as a binary file
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(base64.b64decode(photo_b64))
|
||||
f.write(base64.b64decode(camera_image_b64))
|
||||
|
||||
# Replace the giant base64 string in the dictionary with the local file path
|
||||
# so we don't bloat the MongoDB document
|
||||
data["photo"] = filepath
|
||||
data["camera_image"] = filepath
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": f"Failed to save photo: {str(e)}"}), 500
|
||||
return jsonify({"error": f"Failed to save camera image: {str(e)}"}), 500
|
||||
|
||||
# 2. Save to MongoDB
|
||||
try:
|
||||
@@ -70,6 +70,9 @@ def cooking_params():
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": f"Database error: {str(e)}"}), 500
|
||||
|
||||
# 3. Returns with the cooking parameters
|
||||
|
||||
|
||||
|
||||
@app.route("/device-network", methods=["POST"])
|
||||
|
||||
@@ -14,7 +14,7 @@ RPI_SYSTEMD_SERVICE="smartwave.service"
|
||||
|
||||
# Vérification des arguments
|
||||
if [ -z "$1" ]; then
|
||||
echo "Usage: ./deploy.sh [wifi|lora|rpi|all]"
|
||||
echo "Usage: ./deploy.sh [wifi|mqtt|lora|rpi|all]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -79,6 +79,9 @@ case $CIBLE in
|
||||
"wifi")
|
||||
deploy_to_esp "micro_ondes/esp_wifi" "$PORT_ESP_WIFI" "ESP-WIFI"
|
||||
;;
|
||||
"mqtt")
|
||||
deploy_to_esp "micro_ondes/esp_wifi" "$PORT_ESP_WIFI" "ESP-WIFI"
|
||||
;;
|
||||
"lora")
|
||||
deploy_to_esp "micro_ondes/esp_lora" "$PORT_ESP_LORA" "ESP-LORA"
|
||||
;;
|
||||
@@ -92,6 +95,6 @@ case $CIBLE in
|
||||
# Ajoute les autres ici
|
||||
;;
|
||||
*)
|
||||
echo "Cible inconnue. Utilise 'wifi', 'lora' ou 'all'."
|
||||
echo "Cible inconnue. Utilise 'wifi', 'lora', 'mqtt' ou 'all'."
|
||||
;;
|
||||
esac
|
||||
+12
-1
@@ -1,5 +1,16 @@
|
||||
|
||||
# LoRa
|
||||
|
||||
`mpremote connect /dev/serial/by-id/usb-Silicon_Labs_CP2102_USB_to_UART_Bridge_Controller_0001-if00-port0 repl`
|
||||
`mpremote connect /dev/serial/by-path/pci-0000:00:14.0-usb-0:6.1:1.0-port0 repl`
|
||||
|
||||
# MQTT
|
||||
|
||||
`mpremote connect /dev/serial/by-path/pci-0000:00:14.0-usb-0:6.2:1.0-port0 repl`
|
||||
|
||||
# UART
|
||||
|
||||
| LoRa | MQTT |
|
||||
| --- | --- |
|
||||
| 45 | P17 |
|
||||
| 46 | P16 |
|
||||
| GND | GND |
|
||||
|
||||
@@ -61,7 +61,7 @@ while True:
|
||||
command = uart_device.read()
|
||||
print(f"[Main] Received command from WiFi Board: {command}")
|
||||
|
||||
uart_device.send(f"Hello from esp-32 lora ID {DEVICE_ID}")
|
||||
# uart_device.send(f"Hello from esp-32 lora ID {DEVICE_ID}")
|
||||
|
||||
# 2. Send local metrics over the wire to the WiFi board every few seconds
|
||||
# uart_device.send("Data Pack: LoRa Link RSSI -72dBm")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# This file is executed on every boot (including wake-boot from deepsleep)
|
||||
#import esp
|
||||
#esp.osdebug(None)
|
||||
import esp
|
||||
esp.osdebug(True)
|
||||
#import webrepl
|
||||
#webrepl.start()
|
||||
|
||||
@@ -16,4 +16,4 @@ def do_connect(ssid, pwd):
|
||||
print('network config:', sta_if.ifconfig())
|
||||
|
||||
# Attempt to connect to WiFi network
|
||||
do_connect("Smartwave-1", 'Smartwave-prot-1')
|
||||
# do_connect("Smartwave-1", 'Smartwave-prot-1')
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import _thread
|
||||
import select
|
||||
from machine import Pin
|
||||
from shared import get_mqtt_client, get_uart, config
|
||||
from sensors import temperature_gun
|
||||
from shared import get_mqtt_client, get_uart, config, payloads
|
||||
import time
|
||||
import ujson as json
|
||||
import sys
|
||||
|
||||
# Simple thread-safe queue list
|
||||
msg_queue = []
|
||||
@@ -13,17 +16,22 @@ def queue_publish(topic, payload):
|
||||
with queue_lock:
|
||||
msg_queue.append((topic, payload))
|
||||
|
||||
# --- Hardware & Client Setup ---
|
||||
vext = Pin(19, Pin.OUT)
|
||||
vext.value(0)
|
||||
time.sleep_ms(100)
|
||||
# --- INITIALIZE CAMERA ---
|
||||
try:
|
||||
# Pass your confirmed working SCL and SDA pins here
|
||||
temperature_gun.init_camera(scl_pin=21, sda_pin=22, freq=100000)
|
||||
except Exception as e:
|
||||
print("[Main] Critical: Camera setup failed!")
|
||||
sys.print_exception(e)
|
||||
|
||||
# --- READ DEVICE ID ---
|
||||
try:
|
||||
with open("device_id.txt", "r") as f:
|
||||
DEVICE_ID = f.read().strip()
|
||||
except Exception:
|
||||
DEVICE_ID = "ESP32_Inconnu"
|
||||
|
||||
# --- MQTT SETUP ---
|
||||
MQTT_CA_FILE = "/certs/ca.crt"
|
||||
|
||||
mqtt_client = get_mqtt_client(
|
||||
@@ -34,8 +42,28 @@ mqtt_client = get_mqtt_client(
|
||||
keepalive=config.MQTT_KEEPALIVE,
|
||||
)
|
||||
|
||||
global orchestrator_id
|
||||
orchestrator_id = None
|
||||
def on_mqtt_message(message):
|
||||
print("[MQTT Thread] Received message:", message)
|
||||
|
||||
# Try and parse the payload as json, but if it fails, just print the raw payload
|
||||
payload_data=None
|
||||
try:
|
||||
payload_data = json.loads(message['payload'])
|
||||
except Exception as e:
|
||||
print("[MQTT Thread] Error parsing JSON:", e)
|
||||
sys.print_exception(e)
|
||||
pass # Maybe it's not JSON
|
||||
|
||||
if message['topic'] == config.MQTT_TOPIC_HELLO and payload_data and "id_orchestrator" in payload_data and payload_data["id_microwave"] == DEVICE_ID:
|
||||
print("[MQTT Thread] Hello response received from orchestrator:", payload_data["id_orchestrator"])
|
||||
global orchestrator_id
|
||||
orchestrator_id = payload_data["id_orchestrator"]
|
||||
# Unsubscribe from the hello topic since we got a response
|
||||
mqtt_client.unsubscribe(config.MQTT_TOPIC_HELLO)
|
||||
print("[MQTT Thread] Unsubscribed from topic:", config.MQTT_TOPIC_HELLO)
|
||||
print("[MQTT Thread] Message processing complete.")
|
||||
|
||||
mqtt_client.set_callback(on_mqtt_message)
|
||||
|
||||
@@ -73,7 +101,7 @@ def mqtt_background_thread():
|
||||
|
||||
# 3. Handle Keepalive tracking manually
|
||||
if time.time() - last_check >= 15:
|
||||
print("[Thread] Sending keepalive ping...")
|
||||
# print("[Thread] Sending keepalive ping...")
|
||||
mqtt_client._client.ping()
|
||||
last_check = time.time()
|
||||
|
||||
@@ -82,6 +110,7 @@ def mqtt_background_thread():
|
||||
|
||||
except Exception as e:
|
||||
print("[Thread] Connection dropped or error encountered:", e)
|
||||
sys.print_exception(e)
|
||||
print("[Thread] Cleaning up socket context. Retrying in 5 seconds...")
|
||||
|
||||
# --- FIX FOR ERROR 23 (SOCKET LEAK) ---
|
||||
@@ -99,29 +128,54 @@ def mqtt_background_thread():
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(5)
|
||||
|
||||
# --- UART BACKGROUND THREAD ---
|
||||
def uart_background_thread():
|
||||
"""Background UART worker handling all serial operations safely."""
|
||||
print("[Thread] Background UART worker started.")
|
||||
|
||||
uart_device = get_uart(uart_id=2, tx_pin=17, rx_pin=16)
|
||||
|
||||
while True:
|
||||
try:
|
||||
# 1. Check for incoming messages from the Heltec board
|
||||
while uart_device.any():
|
||||
incoming_msg = uart_device.read()
|
||||
print(f"[Thread] Received from esp-lora over UART: {incoming_msg}")
|
||||
|
||||
# 2. Example: Send data to the Heltec board every 5 seconds
|
||||
# uart_device.send("Status Check: WiFi Active")
|
||||
|
||||
# UART
|
||||
uart_device = get_uart(uart_id=2, tx_pin=17, rx_pin=16)
|
||||
time.sleep(5) # Fast responsive polling loop for local UART
|
||||
|
||||
except Exception as e:
|
||||
print("[Thread] UART error encountered:", e)
|
||||
time.sleep(5)
|
||||
|
||||
# --- Launch background worker ---
|
||||
_thread.start_new_thread(mqtt_background_thread, ())
|
||||
# _thread.start_new_thread(mqtt_background_thread, ())
|
||||
# _thread.start_new_thread(uart_background_thread, ())
|
||||
|
||||
|
||||
# --- MAIN APPLICATION THREAD (Core 0) ---
|
||||
print("[Main] Main execution path active.")
|
||||
time.sleep(2) # Give the thread a moment to initial connect
|
||||
mqtt_hello_sent_timestamp = -config.MQTT_HELLO_INTERVAL
|
||||
mqtt_client.subscribe(config.MQTT_TOPIC_HELLO, qos=config.MQTT_QOS)
|
||||
|
||||
while True:
|
||||
print("[Main] Queueing a test message for MQTT...")
|
||||
# Instead of direct publishing, push it to the queue safely
|
||||
queue_publish(config.MQTT_TOPIC_SENSOR, "Hello from ESP32!")
|
||||
|
||||
# 1. Check if the Heltec V3 sent us something over the wire
|
||||
while uart_device.any():
|
||||
incoming_msg = uart_device.read()
|
||||
print(f"[Main] Received from esp-lora over UART: {incoming_msg}")
|
||||
# MQTT HELLO sent every x seconds until we get a response from the orchestrator
|
||||
if (orchestrator_id == None and -(mqtt_hello_sent_timestamp - time.time()) > config.MQTT_HELLO_INTERVAL):
|
||||
print("[Main] Attempting to send initial hello to orchestrator...")
|
||||
queue_publish(config.MQTT_TOPIC_HELLO, payloads.mqtt_hello(DEVICE_ID))
|
||||
mqtt_hello_sent_timestamp = time.time()
|
||||
pass
|
||||
|
||||
# Sensors
|
||||
print(f"[Main] Reading temperature from the gun sensor...")
|
||||
temp = temperature_gun.read_temperature()
|
||||
print(f"[Main] Temperature read: {temp}°C")
|
||||
|
||||
# 2. Example: Send data to the Heltec board every 5 seconds
|
||||
# uart_device.send("Status Check: WiFi Active")
|
||||
|
||||
time.sleep_ms(200) # Fast responsive polling loop for local UART
|
||||
time.sleep(1)
|
||||
@@ -0,0 +1 @@
|
||||
import sensors.temperature_gun as temperature_gun
|
||||
@@ -0,0 +1,969 @@
|
||||
"""
|
||||
Temperatue gun sensor module
|
||||
using the MLX90640-D55/D110 sensor. This module provides a function to read the temperature from the gun sensor.
|
||||
Resolution of 32x24 pixels,
|
||||
I2C interface
|
||||
Noise Equivalent Temperature difference (NETD) is 0.1K RMS @ 1Hz refresh rate
|
||||
"""
|
||||
|
||||
import machine # type: ignore
|
||||
import math
|
||||
import struct
|
||||
import time
|
||||
from micropython import const# Some libraries that we will use
|
||||
import time
|
||||
|
||||
|
||||
class RefreshRate: # pylint: disable=too-few-public-methods
|
||||
""" Enum-like class for MLX90640's refresh rate """
|
||||
REFRESH_0_5_HZ = const(0b000) # 0.5Hz
|
||||
REFRESH_1_HZ = const(0b001) # 1Hz
|
||||
REFRESH_2_HZ = const(0b010) # 2Hz
|
||||
REFRESH_4_HZ = const(0b011) # 4Hz
|
||||
REFRESH_8_HZ = const(0b100) # 8Hz
|
||||
REFRESH_16_HZ = const(0b101) # 16Hz
|
||||
REFRESH_32_HZ = const(0b110) # 32Hz
|
||||
REFRESH_64_HZ = const(0b111) # 64Hz
|
||||
|
||||
class ContextManaged:
|
||||
"""An object that automatically deinitializes hardware with a context manager."""
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.deinit()
|
||||
|
||||
# pylint: disable=no-self-use
|
||||
def deinit(self):
|
||||
"""Free any hardware used by the object."""
|
||||
return
|
||||
|
||||
class Lockable(ContextManaged):
|
||||
"""An object that must be locked to prevent collisions on a microcontroller resource."""
|
||||
|
||||
_locked = False
|
||||
|
||||
def try_lock(self):
|
||||
"""Attempt to grab the lock. Return True on success, False if the lock is already taken."""
|
||||
if self._locked:
|
||||
return False
|
||||
self._locked = True
|
||||
return True
|
||||
|
||||
def unlock(self):
|
||||
"""Release the lock so others may use the resource."""
|
||||
if self._locked:
|
||||
self._locked = False
|
||||
else:
|
||||
raise ValueError("Not locked")
|
||||
|
||||
class I2C(Lockable):
|
||||
def __init__(self, pins=(21, 22), frequency=100000):
|
||||
self.init(pins, frequency)
|
||||
|
||||
def init(self, pins, frequency):
|
||||
self.deinit()
|
||||
|
||||
# 1. Force the ESP32 to activate its internal pull-up resistors on these pins
|
||||
self._pins = (
|
||||
machine.Pin(int(pins[0]), machine.Pin.IN, machine.Pin.PULL_UP),
|
||||
machine.Pin(int(pins[1]), machine.Pin.IN, machine.Pin.PULL_UP)
|
||||
)
|
||||
|
||||
try:
|
||||
# 2. Bypasses the glitchy ESP32 hardware block using SoftI2C
|
||||
# (Note: SoftI2C does not take a bus ID number like '0')
|
||||
self._i2c = machine.SoftI2C(scl=self._pins[0], sda=self._pins[1], freq=frequency)
|
||||
except RuntimeError:
|
||||
raise
|
||||
print(f"Created resilient SoftI2C: {self._i2c}")
|
||||
|
||||
def deinit(self):
|
||||
try:
|
||||
del self._i2c
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
def scan(self):
|
||||
return self._i2c.scan()
|
||||
|
||||
def readfrom_into(self, address, buffer, *, start=0, end=None):
|
||||
if start is not 0 or end is not None:
|
||||
if end is None:
|
||||
end = len(buffer)
|
||||
buffer = memoryview(buffer)[start:end]
|
||||
stop = True # remove for efficiency later
|
||||
return self._i2c.readfrom_into(address, buffer)
|
||||
|
||||
def writeto(self, address, buffer, *, start=0, end=None, stop=True):
|
||||
if isinstance(buffer, str):
|
||||
buffer = bytes([ord(x) for x in buffer])
|
||||
if start is not 0 or end is not None:
|
||||
if end is None:
|
||||
return self._i2c.writeto(address, memoryview(buffer)[start:], stop)
|
||||
else:
|
||||
return self._i2c.writeto(address, memoryview(buffer)[start:end], stop)
|
||||
return self._i2c.writeto(address, buffer, stop)
|
||||
|
||||
class I2CDevice:
|
||||
def __init__(self, i2c, device_address, probe=True):
|
||||
self.i2c = i2c
|
||||
self._has_write_read = False # hasattr(self.i2c, "writeto_then_readfrom") --> has been turned to False
|
||||
self.device_address = device_address
|
||||
|
||||
if probe:
|
||||
self.__probe_for_device()
|
||||
|
||||
def readinto(self, buf, *, start=0, end=None):
|
||||
if end is None:
|
||||
end = len(buf)
|
||||
self.i2c.readfrom_into(self.device_address, buf, start=start, end=end)
|
||||
|
||||
def write(self, buf, *, start=0, end=None, stop=True):
|
||||
if end is None:
|
||||
end = len(buf)
|
||||
self.i2c.writeto(self.device_address, buf, start=start, end=end, stop=stop)
|
||||
|
||||
# pylint: disable-msg=too-many-arguments
|
||||
def write_then_readinto(
|
||||
self,
|
||||
out_buffer,
|
||||
in_buffer,
|
||||
*,
|
||||
out_start=0,
|
||||
out_end=None,
|
||||
in_start=0,
|
||||
in_end=None,
|
||||
stop=False
|
||||
):
|
||||
if out_end is None:
|
||||
out_end = len(out_buffer)
|
||||
if in_end is None:
|
||||
in_end = len(in_buffer)
|
||||
if stop:
|
||||
raise ValueError("Stop must be False. Use writeto instead.")
|
||||
if self._has_write_read:
|
||||
#print("c",dir(self.i2c))
|
||||
# In linux, at least, this is a special kernel function call
|
||||
self.i2c.writeto_then_readfrom(
|
||||
self.device_address,
|
||||
out_buffer,
|
||||
in_buffer,
|
||||
out_start=out_start,
|
||||
out_end=out_end,
|
||||
in_start=in_start,
|
||||
in_end=in_end,
|
||||
)
|
||||
|
||||
else:
|
||||
# If we don't have a special implementation, we can fake it with two calls
|
||||
self.i2c.writeto(self.device_address, out_buffer, stop=False) # These lines have been changed to make it work with wipy micropython I2C module
|
||||
#self.write(out_buffer, start=out_start, end=out_end, stop=False)
|
||||
#self.readinto(in_buffer, start=in_start, end=in_end)
|
||||
self.i2c.readfrom_into(self.device_address, in_buffer) # These lines have been changed to make it work with wipy micropython I2C module
|
||||
|
||||
|
||||
# pylint: enable-msg=too-many-arguments
|
||||
|
||||
def __enter__(self):
|
||||
while not self.i2c.try_lock():
|
||||
pass
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.i2c.unlock()
|
||||
return False
|
||||
|
||||
def __probe_for_device(self):
|
||||
"""
|
||||
Try to read a byte from an address,
|
||||
if you get an OSError it means the device is not there
|
||||
or that the device does not support these means of probing
|
||||
"""
|
||||
while not self.i2c.try_lock():
|
||||
pass
|
||||
try:
|
||||
self.i2c.writeto(self.device_address, b"")
|
||||
except OSError:
|
||||
# some OS's dont like writing an empty bytesting...
|
||||
# Retry by reading a byte
|
||||
try:
|
||||
result = bytearray(1)
|
||||
self.i2c.readfrom_into(self.device_address, result)
|
||||
except OSError:
|
||||
raise ValueError("No I2C device at address: %x" % self.device_address)
|
||||
finally:
|
||||
self.i2c.unlock()
|
||||
|
||||
eeData = [0] * const(832)
|
||||
I2C_READ_LEN = const(2048)
|
||||
SCALEALPHA = const(0.000001)
|
||||
MLX90640_DEVICEID1 = const(0x2407)
|
||||
OPENAIR_TA_SHIFT = const(8)
|
||||
|
||||
class MLX90640: # pylint: disable=too-many-instance-attributes
|
||||
"""Interface to the MLX90640 temperature sensor."""
|
||||
|
||||
kVdd = 0
|
||||
vdd25 = 0
|
||||
KvPTAT = 0
|
||||
KtPTAT = 0
|
||||
vPTAT25 = 0
|
||||
alphaPTAT = 0
|
||||
gainEE = 0
|
||||
tgc = 0
|
||||
KsTa = 0
|
||||
resolutionEE = 0
|
||||
calibrationModeEE = 0
|
||||
ksTo = [0] * 5
|
||||
ct = [0] * 5
|
||||
alpha = [0] * 768
|
||||
alphaScale = 0
|
||||
offset = [0] * 768
|
||||
kta = [0] * 768
|
||||
ktaScale = 0
|
||||
kv = [0] * 768
|
||||
kvScale = 0
|
||||
cpAlpha = [0] * 2
|
||||
cpOffset = [0] * 2
|
||||
ilChessC = [0] * 3
|
||||
brokenPixels = [0xFFFF] * 5
|
||||
outlierPixels = [0xFFFF] * 5
|
||||
cpKta = 0
|
||||
cpKv = 0
|
||||
|
||||
def __init__(self, i2c_bus, address=0x33):
|
||||
self.i2c_device = I2CDevice(i2c_bus, address)
|
||||
self._I2CReadWords(0x2400, eeData)
|
||||
# print(eeData)
|
||||
self._ExtractParameters()
|
||||
|
||||
@property
|
||||
def serial_number(self):
|
||||
""" 3-item tuple of hex values that are unique to each MLX90640 """
|
||||
serialWords = [0, 0, 0]
|
||||
self._I2CReadWords(MLX90640_DEVICEID1, serialWords)
|
||||
return serialWords
|
||||
|
||||
@property
|
||||
def refresh_rate(self):
|
||||
""" How fast the MLX90640 will spit out data. Start at lowest speed in
|
||||
RefreshRate and then slowly increase I2C clock rate and rate until you
|
||||
max out. The sensor does not like it if the I2C host cannot 'keep up'!"""
|
||||
controlRegister = [0]
|
||||
self._I2CReadWords(0x800D, controlRegister)
|
||||
return (controlRegister[0] >> 7) & 0x07
|
||||
|
||||
@refresh_rate.setter
|
||||
def refresh_rate(self, rate):
|
||||
controlRegister = [0]
|
||||
value = (rate & 0x7) << 7
|
||||
self._I2CReadWords(0x800D, controlRegister)
|
||||
value |= controlRegister[0] & 0xFC7F
|
||||
self._I2CWriteWord(0x800D, value)
|
||||
|
||||
def getFrame(self, framebuf):
|
||||
""" Request both 'halves' of a frame from the sensor, merge them
|
||||
and calculate the temperature in C for each of 32x24 pixels. Placed
|
||||
into the 768-element array passed in! """
|
||||
emissivity = 0.95
|
||||
tr = 23.15
|
||||
mlx90640Frame = [0] * 834
|
||||
|
||||
for _ in range(2):
|
||||
status = self._GetFrameData(mlx90640Frame)
|
||||
if status < 0:
|
||||
raise RuntimeError("Frame data error")
|
||||
# For a MLX90640 in the open air the shift is -8 degC.
|
||||
tr = self._GetTa(mlx90640Frame) - OPENAIR_TA_SHIFT
|
||||
self._CalculateTo(mlx90640Frame, emissivity, tr, framebuf)
|
||||
|
||||
def _GetFrameData(self, frameData):
|
||||
dataReady = 0
|
||||
cnt = 0
|
||||
statusRegister = [0]
|
||||
controlRegister = [0]
|
||||
|
||||
while dataReady == 0:
|
||||
self._I2CReadWords(0x8000, statusRegister)
|
||||
dataReady = statusRegister[0] & 0x0008
|
||||
# print("ready status: 0x%x" % dataReady)
|
||||
|
||||
while (dataReady != 0) and (cnt < 5):
|
||||
self._I2CWriteWord(0x8000, 0x0030)
|
||||
# print("Read frame", cnt)
|
||||
self._I2CReadWords(0x0400, frameData, end=832)
|
||||
|
||||
self._I2CReadWords(0x8000, statusRegister)
|
||||
dataReady = statusRegister[0] & 0x0008
|
||||
# print("frame ready: 0x%x" % dataReady)
|
||||
cnt += 1
|
||||
|
||||
if cnt > 4:
|
||||
raise RuntimeError("Too many retries")
|
||||
|
||||
self._I2CReadWords(0x800D, controlRegister)
|
||||
frameData[832] = controlRegister[0]
|
||||
frameData[833] = statusRegister[0] & 0x0001
|
||||
return frameData[833]
|
||||
|
||||
def _GetTa(self, frameData):
|
||||
vdd = self._GetVdd(frameData)
|
||||
|
||||
ptat = frameData[800]
|
||||
if ptat > 32767:
|
||||
ptat -= 65536
|
||||
|
||||
ptatArt = frameData[768]
|
||||
if ptatArt > 32767:
|
||||
ptatArt -= 65536
|
||||
ptatArt = (ptat / (ptat * self.alphaPTAT + ptatArt)) * math.pow(2, 18)
|
||||
|
||||
ta = ptatArt / (1 + self.KvPTAT * (vdd - 3.3)) - self.vPTAT25
|
||||
ta = ta / self.KtPTAT + 25
|
||||
return ta
|
||||
|
||||
def _GetVdd(self, frameData):
|
||||
vdd = frameData[810]
|
||||
if vdd > 32767:
|
||||
vdd -= 65536
|
||||
|
||||
resolutionRAM = (frameData[832] & 0x0C00) >> 10
|
||||
resolutionCorrection = math.pow(2, self.resolutionEE) / math.pow(
|
||||
2, resolutionRAM
|
||||
)
|
||||
vdd = (resolutionCorrection * vdd - self.vdd25) / self.kVdd + 3.3
|
||||
|
||||
return vdd
|
||||
|
||||
def _CalculateTo(self, frameData, emissivity, tr, result):
|
||||
# pylint: disable=too-many-locals, too-many-branches, too-many-statements
|
||||
subPage = frameData[833]
|
||||
alphaCorrR = [0] * 4
|
||||
irDataCP = [0, 0]
|
||||
|
||||
vdd = self._GetVdd(frameData)
|
||||
ta = self._GetTa(frameData)
|
||||
|
||||
ta4 = ta + 273.15
|
||||
ta4 = ta4 * ta4
|
||||
ta4 = ta4 * ta4
|
||||
tr4 = tr + 273.15
|
||||
tr4 = tr4 * tr4
|
||||
tr4 = tr4 * tr4
|
||||
taTr = tr4 - (tr4 - ta4) / emissivity
|
||||
|
||||
ktaScale = math.pow(2, self.ktaScale)
|
||||
kvScale = math.pow(2, self.kvScale)
|
||||
alphaScale = math.pow(2, self.alphaScale)
|
||||
|
||||
alphaCorrR[0] = 1 / (1 + self.ksTo[0] * 40)
|
||||
alphaCorrR[1] = 1
|
||||
alphaCorrR[2] = 1 + self.ksTo[1] * self.ct[2]
|
||||
alphaCorrR[3] = alphaCorrR[2] * (1 + self.ksTo[2] * (self.ct[3] - self.ct[2]))
|
||||
|
||||
# --------- Gain calculation -----------------------------------
|
||||
gain = frameData[778]
|
||||
if gain > 32767:
|
||||
gain -= 65536
|
||||
gain = self.gainEE / gain
|
||||
|
||||
# --------- To calculation -------------------------------------
|
||||
mode = (frameData[832] & 0x1000) >> 5
|
||||
|
||||
irDataCP[0] = frameData[776]
|
||||
irDataCP[1] = frameData[808]
|
||||
for i in range(2):
|
||||
if irDataCP[i] > 32767:
|
||||
irDataCP[i] -= 65536
|
||||
irDataCP[i] *= gain
|
||||
|
||||
irDataCP[0] -= (
|
||||
self.cpOffset[0]
|
||||
* (1 + self.cpKta * (ta - 25))
|
||||
* (1 + self.cpKv * (vdd - 3.3))
|
||||
)
|
||||
if mode == self.calibrationModeEE:
|
||||
irDataCP[1] -= (
|
||||
self.cpOffset[1]
|
||||
* (1 + self.cpKta * (ta - 25))
|
||||
* (1 + self.cpKv * (vdd - 3.3))
|
||||
)
|
||||
else:
|
||||
irDataCP[1] -= (
|
||||
(self.cpOffset[1] + self.ilChessC[0])
|
||||
* (1 + self.cpKta * (ta - 25))
|
||||
* (1 + self.cpKv * (vdd - 3.3))
|
||||
)
|
||||
|
||||
for pixelNumber in range(768):
|
||||
ilPattern = pixelNumber // 32 - (pixelNumber // 64) * 2
|
||||
chessPattern = ilPattern ^ (pixelNumber - (pixelNumber // 2) * 2)
|
||||
conversionPattern = (
|
||||
(pixelNumber + 2) // 4
|
||||
- (pixelNumber + 3) // 4
|
||||
+ (pixelNumber + 1) // 4
|
||||
- pixelNumber // 4
|
||||
) * (1 - 2 * ilPattern)
|
||||
|
||||
if mode == 0:
|
||||
pattern = ilPattern
|
||||
else:
|
||||
pattern = chessPattern
|
||||
|
||||
if pattern == frameData[833]:
|
||||
irData = frameData[pixelNumber]
|
||||
if irData > 32767:
|
||||
irData -= 65536
|
||||
irData *= gain
|
||||
|
||||
kta = self.kta[pixelNumber] / ktaScale
|
||||
kv = self.kv[pixelNumber] / kvScale
|
||||
irData -= (
|
||||
self.offset[pixelNumber]
|
||||
* (1 + kta * (ta - 25))
|
||||
* (1 + kv * (vdd - 3.3))
|
||||
)
|
||||
|
||||
if mode != self.calibrationModeEE:
|
||||
irData += (
|
||||
self.ilChessC[2] * (2 * ilPattern - 1)
|
||||
- self.ilChessC[1] * conversionPattern
|
||||
)
|
||||
|
||||
irData = irData - self.tgc * irDataCP[subPage]
|
||||
irData /= emissivity
|
||||
|
||||
alphaCompensated = SCALEALPHA * alphaScale / self.alpha[pixelNumber]
|
||||
alphaCompensated *= 1 + self.KsTa * (ta - 25)
|
||||
|
||||
Sx = (
|
||||
alphaCompensated
|
||||
* alphaCompensated
|
||||
* alphaCompensated
|
||||
* (irData + alphaCompensated * taTr)
|
||||
)
|
||||
Sx = math.sqrt(math.sqrt(Sx)) * self.ksTo[1]
|
||||
|
||||
To = (
|
||||
math.sqrt(
|
||||
math.sqrt(
|
||||
irData
|
||||
/ (alphaCompensated * (1 - self.ksTo[1] * 273.15) + Sx)
|
||||
+ taTr
|
||||
)
|
||||
)
|
||||
- 273.15
|
||||
)
|
||||
|
||||
if To < self.ct[1]:
|
||||
torange = 0
|
||||
elif To < self.ct[2]:
|
||||
torange = 1
|
||||
elif To < self.ct[3]:
|
||||
torange = 2
|
||||
else:
|
||||
torange = 3
|
||||
|
||||
To = (
|
||||
math.sqrt(
|
||||
math.sqrt(
|
||||
irData
|
||||
/ (
|
||||
alphaCompensated
|
||||
* alphaCorrR[torange]
|
||||
* (1 + self.ksTo[torange] * (To - self.ct[torange]))
|
||||
)
|
||||
+ taTr
|
||||
)
|
||||
)
|
||||
- 273.15
|
||||
)
|
||||
|
||||
result[pixelNumber] = To
|
||||
|
||||
# pylint: enable=too-many-locals, too-many-branches, too-many-statements
|
||||
|
||||
def _ExtractParameters(self):
|
||||
self._ExtractVDDParameters()
|
||||
self._ExtractPTATParameters()
|
||||
self._ExtractGainParameters()
|
||||
self._ExtractTgcParameters()
|
||||
self._ExtractResolutionParameters()
|
||||
self._ExtractKsTaParameters()
|
||||
self._ExtractKsToParameters()
|
||||
self._ExtractCPParameters()
|
||||
self._ExtractAlphaParameters()
|
||||
self._ExtractOffsetParameters()
|
||||
self._ExtractKtaPixelParameters()
|
||||
self._ExtractKvPixelParameters()
|
||||
self._ExtractCILCParameters()
|
||||
self._ExtractDeviatingPixels()
|
||||
|
||||
def _ExtractVDDParameters(self):
|
||||
# extract VDD
|
||||
self.kVdd = (eeData[51] & 0xFF00) >> 8
|
||||
if self.kVdd > 127:
|
||||
self.kVdd -= 256 # convert to signed
|
||||
self.kVdd *= 32
|
||||
self.vdd25 = eeData[51] & 0x00FF
|
||||
self.vdd25 = ((self.vdd25 - 256) << 5) - 8192
|
||||
|
||||
def _ExtractPTATParameters(self):
|
||||
# extract PTAT
|
||||
self.KvPTAT = (eeData[50] & 0xFC00) >> 10
|
||||
if self.KvPTAT > 31:
|
||||
self.KvPTAT -= 64
|
||||
self.KvPTAT /= 4096
|
||||
self.KtPTAT = eeData[50] & 0x03FF
|
||||
if self.KtPTAT > 511:
|
||||
self.KtPTAT -= 1024
|
||||
self.KtPTAT /= 8
|
||||
self.vPTAT25 = eeData[49]
|
||||
self.alphaPTAT = (eeData[16] & 0xF000) / math.pow(2, 14) + 8
|
||||
|
||||
def _ExtractGainParameters(self):
|
||||
# extract Gain
|
||||
self.gainEE = eeData[48]
|
||||
if self.gainEE > 32767:
|
||||
self.gainEE -= 65536
|
||||
|
||||
def _ExtractTgcParameters(self):
|
||||
# extract Tgc
|
||||
#print(eeData[60])
|
||||
self.tgc = eeData[60] & 0x00FF
|
||||
#print(self.tgc)
|
||||
if self.tgc > 127:
|
||||
self.tgc -= 256
|
||||
self.tgc /= 32
|
||||
#print(self.tgc)
|
||||
|
||||
def _ExtractResolutionParameters(self):
|
||||
# extract resolution
|
||||
self.resolutionEE = (eeData[56] & 0x3000) >> 12
|
||||
|
||||
def _ExtractKsTaParameters(self):
|
||||
# extract KsTa
|
||||
self.KsTa = (eeData[60] & 0xFF00) >> 8
|
||||
if self.KsTa > 127:
|
||||
self.KsTa -= 256
|
||||
self.KsTa /= 8192
|
||||
|
||||
def _ExtractKsToParameters(self):
|
||||
# extract ksTo
|
||||
step = ((eeData[63] & 0x3000) >> 12) * 10
|
||||
self.ct[0] = -40
|
||||
self.ct[1] = 0
|
||||
self.ct[2] = (eeData[63] & 0x00F0) >> 4
|
||||
self.ct[3] = (eeData[63] & 0x0F00) >> 8
|
||||
self.ct[2] *= step
|
||||
self.ct[3] = self.ct[2] + self.ct[3] * step
|
||||
|
||||
KsToScale = (eeData[63] & 0x000F) + 8
|
||||
KsToScale = 1 << KsToScale
|
||||
|
||||
self.ksTo[0] = eeData[61] & 0x00FF
|
||||
self.ksTo[1] = (eeData[61] & 0xFF00) >> 8
|
||||
self.ksTo[2] = eeData[62] & 0x00FF
|
||||
self.ksTo[3] = (eeData[62] & 0xFF00) >> 8
|
||||
|
||||
for i in range(4):
|
||||
if self.ksTo[i] > 127:
|
||||
self.ksTo[i] -= 256
|
||||
self.ksTo[i] /= KsToScale
|
||||
self.ksTo[4] = -0.0002
|
||||
|
||||
def _ExtractCPParameters(self):
|
||||
# extract CP
|
||||
offsetSP = [0] * 2
|
||||
alphaSP = [0] * 2
|
||||
|
||||
alphaScale = ((eeData[32] & 0xF000) >> 12) + 27
|
||||
|
||||
offsetSP[0] = eeData[58] & 0x03FF
|
||||
if offsetSP[0] > 511:
|
||||
offsetSP[0] -= 1024
|
||||
|
||||
offsetSP[1] = (eeData[58] & 0xFC00) >> 10
|
||||
if offsetSP[1] > 31:
|
||||
offsetSP[1] -= 64
|
||||
offsetSP[1] += offsetSP[0]
|
||||
|
||||
alphaSP[0] = eeData[57] & 0x03FF
|
||||
if alphaSP[0] > 511:
|
||||
alphaSP[0] -= 1024
|
||||
alphaSP[0] /= math.pow(2, alphaScale)
|
||||
|
||||
alphaSP[1] = (eeData[57] & 0xFC00) >> 10
|
||||
if alphaSP[1] > 31:
|
||||
alphaSP[1] -= 64
|
||||
alphaSP[1] = (1 + alphaSP[1] / 128) * alphaSP[0]
|
||||
|
||||
cpKta = eeData[59] & 0x00FF
|
||||
if cpKta > 127:
|
||||
cpKta -= 256
|
||||
ktaScale1 = ((eeData[56] & 0x00F0) >> 4) + 8
|
||||
self.cpKta = cpKta / math.pow(2, ktaScale1)
|
||||
|
||||
cpKv = (eeData[59] & 0xFF00) >> 8
|
||||
if cpKv > 127:
|
||||
cpKv -= 256
|
||||
kvScale = (eeData[56] & 0x0F00) >> 8
|
||||
self.cpKv = cpKv / math.pow(2, kvScale)
|
||||
|
||||
self.cpAlpha[0] = alphaSP[0]
|
||||
self.cpAlpha[1] = alphaSP[1]
|
||||
self.cpOffset[0] = offsetSP[0]
|
||||
self.cpOffset[1] = offsetSP[1]
|
||||
#print(self.cpAlpha[0])
|
||||
#print(self.cpAlpha[1])
|
||||
|
||||
def _ExtractAlphaParameters(self):
|
||||
# extract alpha
|
||||
accRemScale = eeData[32] & 0x000F
|
||||
accColumnScale = (eeData[32] & 0x00F0) >> 4
|
||||
accRowScale = (eeData[32] & 0x0F00) >> 8
|
||||
alphaScale = ((eeData[32] & 0xF000) >> 12) + 30
|
||||
alphaRef = eeData[33]
|
||||
accRow = [0] * 24
|
||||
accColumn = [0] * 32
|
||||
alphaTemp = [0] * 768
|
||||
|
||||
for i in range(6):
|
||||
p = i * 4
|
||||
accRow[p + 0] = eeData[34 + i] & 0x000F
|
||||
accRow[p + 1] = (eeData[34 + i] & 0x00F0) >> 4
|
||||
accRow[p + 2] = (eeData[34 + i] & 0x0F00) >> 8
|
||||
accRow[p + 3] = (eeData[34 + i] & 0xF000) >> 12
|
||||
|
||||
for i in range(24):
|
||||
if accRow[i] > 7:
|
||||
accRow[i] -= 16
|
||||
|
||||
for i in range(8):
|
||||
p = i * 4
|
||||
accColumn[p + 0] = eeData[40 + i] & 0x000F
|
||||
accColumn[p + 1] = (eeData[40 + i] & 0x00F0) >> 4
|
||||
accColumn[p + 2] = (eeData[40 + i] & 0x0F00) >> 8
|
||||
accColumn[p + 3] = (eeData[40 + i] & 0xF000) >> 12
|
||||
|
||||
for i in range(32):
|
||||
if accColumn[i] > 7:
|
||||
accColumn[i] -= 16
|
||||
for i in range(24):
|
||||
for j in range(32):
|
||||
p = 32 * i + j
|
||||
alphaTemp[p] = (eeData[64 + p] & 0x03F0) >> 4
|
||||
if alphaTemp[p] > 31:
|
||||
alphaTemp[p] -= 64
|
||||
alphaTemp[p] *= 1 << accRemScale
|
||||
alphaTemp[p] += (
|
||||
alphaRef
|
||||
+ (accRow[i] << accRowScale)
|
||||
+ (accColumn[j] << accColumnScale)
|
||||
)
|
||||
alphaTemp[p] /= math.pow(2, alphaScale)
|
||||
alphaTemp[p] -= self.tgc * (self.cpAlpha[0] + self.cpAlpha[1]) / 2
|
||||
alphaTemp[p] = SCALEALPHA / alphaTemp[p]
|
||||
# print("alphaTemp: ", alphaTemp)
|
||||
|
||||
temp = max(alphaTemp)
|
||||
#print("temp", temp)
|
||||
|
||||
alphaScale = 0
|
||||
while temp < 32768:
|
||||
temp *= 2
|
||||
alphaScale += 1
|
||||
|
||||
for i in range(768):
|
||||
temp = alphaTemp[i] * math.pow(2, alphaScale)
|
||||
self.alpha[i] = int(temp + 0.5)
|
||||
|
||||
self.alphaScale = alphaScale
|
||||
|
||||
def _ExtractOffsetParameters(self):
|
||||
# extract offset
|
||||
occRow = [0] * 24
|
||||
occColumn = [0] * 32
|
||||
|
||||
occRemScale = eeData[16] & 0x000F
|
||||
occColumnScale = (eeData[16] & 0x00F0) >> 4
|
||||
occRowScale = (eeData[16] & 0x0F00) >> 8
|
||||
offsetRef = eeData[17]
|
||||
if offsetRef > 32767:
|
||||
offsetRef -= 65536
|
||||
|
||||
for i in range(6):
|
||||
p = i * 4
|
||||
occRow[p + 0] = eeData[18 + i] & 0x000F
|
||||
occRow[p + 1] = (eeData[18 + i] & 0x00F0) >> 4
|
||||
occRow[p + 2] = (eeData[18 + i] & 0x0F00) >> 8
|
||||
occRow[p + 3] = (eeData[18 + i] & 0xF000) >> 12
|
||||
|
||||
for i in range(24):
|
||||
if occRow[i] > 7:
|
||||
occRow[i] -= 16
|
||||
|
||||
for i in range(8):
|
||||
p = i * 4
|
||||
occColumn[p + 0] = eeData[24 + i] & 0x000F
|
||||
occColumn[p + 1] = (eeData[24 + i] & 0x00F0) >> 4
|
||||
occColumn[p + 2] = (eeData[24 + i] & 0x0F00) >> 8
|
||||
occColumn[p + 3] = (eeData[24 + i] & 0xF000) >> 12
|
||||
|
||||
for i in range(32):
|
||||
if occColumn[i] > 7:
|
||||
occColumn[i] -= 16
|
||||
|
||||
for i in range(24):
|
||||
for j in range(32):
|
||||
p = 32 * i + j
|
||||
self.offset[p] = (eeData[64 + p] & 0xFC00) >> 10
|
||||
if self.offset[p] > 31:
|
||||
self.offset[p] -= 64
|
||||
self.offset[p] *= 1 << occRemScale
|
||||
self.offset[p] += (
|
||||
offsetRef
|
||||
+ (occRow[i] << occRowScale)
|
||||
+ (occColumn[j] << occColumnScale)
|
||||
)
|
||||
|
||||
def _ExtractKtaPixelParameters(self): # pylint: disable=too-many-locals
|
||||
# extract KtaPixel
|
||||
KtaRC = [0] * 4
|
||||
ktaTemp = [0] * 768
|
||||
|
||||
KtaRoCo = (eeData[54] & 0xFF00) >> 8
|
||||
if KtaRoCo > 127:
|
||||
KtaRoCo -= 256
|
||||
KtaRC[0] = KtaRoCo
|
||||
|
||||
KtaReCo = eeData[54] & 0x00FF
|
||||
if KtaReCo > 127:
|
||||
KtaReCo -= 256
|
||||
KtaRC[2] = KtaReCo
|
||||
|
||||
KtaRoCe = (eeData[55] & 0xFF00) >> 8
|
||||
if KtaRoCe > 127:
|
||||
KtaRoCe -= 256
|
||||
KtaRC[1] = KtaRoCe
|
||||
|
||||
KtaReCe = eeData[55] & 0x00FF
|
||||
if KtaReCe > 127:
|
||||
KtaReCe -= 256
|
||||
KtaRC[3] = KtaReCe
|
||||
|
||||
ktaScale1 = ((eeData[56] & 0x00F0) >> 4) + 8
|
||||
ktaScale2 = eeData[56] & 0x000F
|
||||
|
||||
for i in range(24):
|
||||
for j in range(32):
|
||||
p = 32 * i + j
|
||||
split = 2 * (p // 32 - (p // 64) * 2) + p % 2
|
||||
ktaTemp[p] = (eeData[64 + p] & 0x000E) >> 1
|
||||
if ktaTemp[p] > 3:
|
||||
ktaTemp[p] -= 8
|
||||
ktaTemp[p] *= 1 << ktaScale2
|
||||
ktaTemp[p] += KtaRC[split]
|
||||
ktaTemp[p] /= math.pow(2, ktaScale1)
|
||||
# ktaTemp[p] = ktaTemp[p] * mlx90640->offset[p];
|
||||
|
||||
temp = abs(ktaTemp[0])
|
||||
for kta in ktaTemp:
|
||||
temp = max(temp, abs(kta))
|
||||
|
||||
ktaScale1 = 0
|
||||
while temp < 64:
|
||||
temp *= 2
|
||||
ktaScale1 += 1
|
||||
|
||||
for i in range(768):
|
||||
temp = ktaTemp[i] * math.pow(2, ktaScale1)
|
||||
if temp < 0:
|
||||
self.kta[i] = int(temp - 0.5)
|
||||
else:
|
||||
self.kta[i] = int(temp + 0.5)
|
||||
self.ktaScale = ktaScale1
|
||||
|
||||
def _ExtractKvPixelParameters(self):
|
||||
KvT = [0] * 4
|
||||
kvTemp = [0] * 768
|
||||
|
||||
KvRoCo = (eeData[52] & 0xF000) >> 12
|
||||
if KvRoCo > 7:
|
||||
KvRoCo -= 16
|
||||
KvT[0] = KvRoCo
|
||||
|
||||
KvReCo = (eeData[52] & 0x0F00) >> 8
|
||||
if KvReCo > 7:
|
||||
KvReCo -= 16
|
||||
KvT[2] = KvReCo
|
||||
|
||||
KvRoCe = (eeData[52] & 0x00F0) >> 4
|
||||
if KvRoCe > 7:
|
||||
KvRoCe -= 16
|
||||
KvT[1] = KvRoCe
|
||||
|
||||
KvReCe = eeData[52] & 0x000F
|
||||
if KvReCe > 7:
|
||||
KvReCe -= 16
|
||||
KvT[3] = KvReCe
|
||||
|
||||
kvScale = (eeData[56] & 0x0F00) >> 8
|
||||
|
||||
for i in range(24):
|
||||
for j in range(32):
|
||||
p = 32 * i + j
|
||||
split = 2 * (p // 32 - (p // 64) * 2) + p % 2
|
||||
kvTemp[p] = KvT[split]
|
||||
kvTemp[p] /= math.pow(2, kvScale)
|
||||
# kvTemp[p] = kvTemp[p] * mlx90640->offset[p];
|
||||
|
||||
temp = abs(kvTemp[0])
|
||||
for kv in kvTemp:
|
||||
temp = max(temp, abs(kv))
|
||||
|
||||
kvScale = 0
|
||||
while temp < 64:
|
||||
temp *= 2
|
||||
kvScale += 1
|
||||
|
||||
for i in range(768):
|
||||
temp = kvTemp[i] * math.pow(2, kvScale)
|
||||
if temp < 0:
|
||||
self.kv[i] = int(temp - 0.5)
|
||||
else:
|
||||
self.kv[i] = int(temp + 0.5)
|
||||
self.kvScale = kvScale
|
||||
|
||||
def _ExtractCILCParameters(self):
|
||||
ilChessC = [0] * 3
|
||||
|
||||
self.calibrationModeEE = (eeData[10] & 0x0800) >> 4
|
||||
self.calibrationModeEE = self.calibrationModeEE ^ 0x80
|
||||
|
||||
ilChessC[0] = eeData[53] & 0x003F
|
||||
if ilChessC[0] > 31:
|
||||
ilChessC[0] -= 64
|
||||
ilChessC[0] /= 16.0
|
||||
|
||||
ilChessC[1] = (eeData[53] & 0x07C0) >> 6
|
||||
if ilChessC[1] > 15:
|
||||
ilChessC[1] -= 32
|
||||
ilChessC[1] /= 2.0
|
||||
|
||||
ilChessC[2] = (eeData[53] & 0xF800) >> 11
|
||||
if ilChessC[2] > 15:
|
||||
ilChessC[2] -= 32
|
||||
ilChessC[2] /= 8.0
|
||||
|
||||
self.ilChessC = ilChessC
|
||||
|
||||
def _ExtractDeviatingPixels(self):
|
||||
self.brokenPixels = [0xFFFF] * 5
|
||||
self.outlierPixels = [0xFFFF] * 5
|
||||
|
||||
pixCnt = 0
|
||||
brokenPixCnt = 0
|
||||
outlierPixCnt = 0
|
||||
|
||||
while (pixCnt < 768) and (brokenPixCnt < 5) and (outlierPixCnt < 5):
|
||||
if eeData[pixCnt + 64] == 0:
|
||||
self.brokenPixels[brokenPixCnt] = pixCnt
|
||||
brokenPixCnt += 1
|
||||
elif (eeData[pixCnt + 64] & 0x0001) != 0:
|
||||
self.outlierPixels[outlierPixCnt] = pixCnt
|
||||
outlierPixCnt += 1
|
||||
pixCnt += 1
|
||||
|
||||
if brokenPixCnt > 4:
|
||||
raise RuntimeError("More than 4 broken pixels")
|
||||
if outlierPixCnt > 4:
|
||||
raise RuntimeError("More than 4 outlier pixels")
|
||||
if (brokenPixCnt + outlierPixCnt) > 4:
|
||||
raise RuntimeError("More than 4 faulty pixels")
|
||||
# print("Found %d broken pixels, %d outliers" % (brokenPixCnt, outlierPixCnt))
|
||||
# TODO INCOMPLETE
|
||||
|
||||
def _I2CWriteWord(self, writeAddress, data):
|
||||
cmd = bytearray(4)
|
||||
cmd[0] = writeAddress >> 8
|
||||
cmd[1] = writeAddress & 0x00FF
|
||||
cmd[2] = data >> 8
|
||||
cmd[3] = data & 0x00FF
|
||||
dataCheck = [0]
|
||||
|
||||
with self.i2c_device as i2c:
|
||||
i2c.write(cmd)
|
||||
# print("Wrote:", [hex(i) for i in cmd])
|
||||
time.sleep(0.001)
|
||||
self._I2CReadWords(writeAddress, dataCheck)
|
||||
# print("dataCheck: 0x%x" % dataCheck[0])
|
||||
# if (dataCheck != data):
|
||||
# return -2
|
||||
|
||||
_inbuf = bytearray(2 * I2C_READ_LEN)
|
||||
|
||||
def _I2CReadWords(self, addr, buffer, *, end=None):
|
||||
# stamp = time.monotonic()
|
||||
if end is None:
|
||||
remainingWords = len(buffer)
|
||||
else:
|
||||
remainingWords = end
|
||||
offset = 0
|
||||
addrbuf = bytearray(2)
|
||||
# inbuf = bytearray(2 * I2C_READ_LEN)
|
||||
inbuf = self._inbuf
|
||||
|
||||
with self.i2c_device as i2c:
|
||||
while remainingWords:
|
||||
addrbuf[0] = addr >> 8 # MSB
|
||||
addrbuf[1] = addr & 0xFF # LSB
|
||||
read_words = min(remainingWords, I2C_READ_LEN)
|
||||
i2c.write_then_readinto(
|
||||
addrbuf, inbuf, in_end=read_words * 2
|
||||
) # in bytes
|
||||
# print("-> ", [hex(i) for i in addrbuf])
|
||||
|
||||
outwords = struct.unpack(
|
||||
">" + "H" * read_words, inbuf[0 : read_words * 2]
|
||||
)
|
||||
# print("<- (", read_words, ")", [hex(i) for i in outwords])
|
||||
for i, w in enumerate(outwords):
|
||||
buffer[offset + i] = w
|
||||
offset += read_words
|
||||
remainingWords -= read_words
|
||||
addr += read_words
|
||||
|
||||
ixc = None
|
||||
mlx = None
|
||||
frame = None
|
||||
|
||||
def init_camera(scl_pin=22, sda_pin=21, freq=100000):
|
||||
"""Explicitly initializes the I2C bus and camera after power is stable."""
|
||||
global ixc, mlx, frame
|
||||
|
||||
print(f"[Camera] Initializing I2C on SCL:{scl_pin}, SDA:{sda_pin} at {freq}Hz...")
|
||||
ixc = I2C(pins=(scl_pin, sda_pin), frequency=freq)
|
||||
|
||||
print("[Camera] Probing for MLX90640...")
|
||||
mlx = MLX90640(ixc)
|
||||
|
||||
# Bonus: Your wiki snapshot recommends 16Hz for smooth images!
|
||||
mlx.refresh_rate = RefreshRate.REFRESH_16_HZ
|
||||
|
||||
frame = [0] * 768
|
||||
print("[Camera] Setup successful!")
|
||||
|
||||
def read_temperature():
|
||||
if mlx is None:
|
||||
print("[Camera] Error: Camera not initialized. Call init_camera() first.")
|
||||
return None
|
||||
try:
|
||||
print("Querying camera...")
|
||||
mlx.getFrame(frame)
|
||||
return frame
|
||||
except Exception as e:
|
||||
print(f"[Camera] Read error: {e}")
|
||||
return None
|
||||
@@ -5,7 +5,7 @@ SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
REPO_ROOT=$(dirname -- "$SCRIPT_DIR")
|
||||
PYTHON_BIN="${PYTHON_BIN:-python3}"
|
||||
PYTHON_SCRIPT="${1:-$SCRIPT_DIR/main.py}"
|
||||
REQUIREMENTS_FILE="$REPO_ROOT/requirements.txt"
|
||||
REQUIREMENTS_FILE="$SCRIPT_DIR/requirements.txt"
|
||||
|
||||
if [ ! -f "$PYTHON_SCRIPT" ]; then
|
||||
echo "Python script not found: $PYTHON_SCRIPT" >&2
|
||||
@@ -19,7 +19,7 @@ cd "$SCRIPT_DIR"
|
||||
|
||||
# 2. On lance Docker en arrière-plan
|
||||
echo "Démarrage des conteneurs Docker..."
|
||||
docker compose pull
|
||||
# docker compose pull
|
||||
docker compose up -d --remove-orphans
|
||||
|
||||
# 3. Installation des dépendances (sans '--user' si on est déjà root sous systemd)
|
||||
|
||||
+101
-24
@@ -1,7 +1,12 @@
|
||||
import json
|
||||
import threading
|
||||
import queue
|
||||
import time
|
||||
from shared import get_lora, get_mqtt_client, deviceTypes, config
|
||||
import traceback
|
||||
from orchestrateur.sensors import gps
|
||||
from shared import get_lora, get_mqtt_client, deviceTypes, config, payloads
|
||||
from shared.logging import log
|
||||
from sensors import ultrasonicRanger, temp_hum, button, camera
|
||||
|
||||
# --- Read Unique Device ID ---
|
||||
try:
|
||||
@@ -49,6 +54,7 @@ mqtt_client = get_mqtt_client(
|
||||
)
|
||||
mqtt_client.connect()
|
||||
mqtt_client.subscribe(config.MQTT_TOPIC_SENSOR, qos=config.MQTT_QOS)
|
||||
mqtt_client.subscribe(config.MQTT_TOPIC_HELLO, qos=config.MQTT_QOS)
|
||||
print(f"Subscribed to topic: {config.MQTT_TOPIC_SENSOR}")
|
||||
|
||||
# --- THE CRUCIAL PAHO FIX ---
|
||||
@@ -66,51 +72,122 @@ def mqtt_listener():
|
||||
message = mqtt_client.get_message()
|
||||
|
||||
if message:
|
||||
# Try to parse the payload as a python dictionary, but if it fails, just print the raw payload
|
||||
try:
|
||||
payload = json.loads(message['payload'])
|
||||
except Exception as e:
|
||||
print(f"Error parsing MQTT payload: {e}")
|
||||
payload = message['payload'] # Fallback to raw payload if parsing fails
|
||||
|
||||
print(f"\n[Thread MQTT] Message reçu : {message}")
|
||||
data_queue.put({"source": "MQTT", "data": message})
|
||||
data_queue.put({"source": "MQTT", "topic": message['topic'] ,"data": payload})
|
||||
|
||||
# --- THE CPU FIX ---
|
||||
# Sleep for 100ms. Prevents the thread from turning into an infinite 100% CPU hog.
|
||||
time.sleep(0.1)
|
||||
time.sleep(0.2)
|
||||
|
||||
# Button
|
||||
button_state = False
|
||||
def button_callback():
|
||||
global button_state
|
||||
button_state = not button_state
|
||||
print(f"\n[Thread Button] Button state changed to: {button_state}")
|
||||
|
||||
button.set_callback(button_callback)
|
||||
|
||||
# Launch background monitoring workers
|
||||
threading.Thread(target=lora_listener, daemon=True).start()
|
||||
threading.Thread(target=mqtt_listener, daemon=True).start()
|
||||
# threading.Thread(target=lora_listener, daemon=True).start()
|
||||
# threading.Thread(target=mqtt_listener, daemon=True).start()
|
||||
# Launch button monitoring thread
|
||||
button.start_button_monitoring_thread()
|
||||
|
||||
print("Orchestrateur prêt. Le main loop est libre.")
|
||||
|
||||
# Sensor reading
|
||||
def read_sensors():
|
||||
"""Read all sensors and return a dictionary of their values."""
|
||||
log("\nLecture des capteurs...")
|
||||
sensor_data = {}
|
||||
|
||||
# Read Ultrasonic Ranger
|
||||
distance = ultrasonicRanger.get_dish_height()
|
||||
if distance is not None:
|
||||
log(f"\nLecture du capteur Ultrason : {distance}")
|
||||
sensor_data["ultrasonic_distance"] = distance
|
||||
|
||||
# Read Temperature and Humidity
|
||||
temperature, humidity = temp_hum.get_temperature_and_humidity()
|
||||
if temperature is not None and humidity is not None:
|
||||
log(f"\nLecture du capteur Temp/Hum : {temperature}, {humidity}")
|
||||
sensor_data["temperature"] = temperature
|
||||
sensor_data["humidity"] = humidity
|
||||
|
||||
|
||||
# Read GPS Data
|
||||
gps_data = gps.get_gps_data()
|
||||
if gps_data:
|
||||
log(f"\nLecture du capteur GPS : {gps_data}")
|
||||
sensor_data["gps"] = gps_data
|
||||
|
||||
# Camera
|
||||
picture_bytes = None
|
||||
try:
|
||||
picture_bytes = camera.get_picture()
|
||||
log(f"\nLecture du capteur Caméra : {len(picture_bytes)} bytes")
|
||||
sensor_data["camera_image"] = picture_bytes
|
||||
except Exception as e:
|
||||
log(f"Error reading camera data: {e}")
|
||||
|
||||
# Read Button State (last because he can still change state while reading other sensors)
|
||||
sensor_data["button_state"] = button_state
|
||||
|
||||
return sensor_data
|
||||
|
||||
# --- MAIN EXECUTION LOOP ---
|
||||
while True:
|
||||
try:
|
||||
# Check for non-heartbeat data safely
|
||||
# Check for non-heartbeat data
|
||||
try:
|
||||
msg = data_queue.get(block=False)
|
||||
print(f"\n[Main Loop] Données traitées : {msg['data']}")
|
||||
|
||||
# print(msg)
|
||||
|
||||
if msg["source"] == "LoRa":
|
||||
print(f"\n[Main Loop] LoRa : Données traitées : {msg['data']}")
|
||||
elif msg["source"] == "MQTT":
|
||||
if (msg["topic"] == config.MQTT_TOPIC_HELLO.decode('utf-8')):
|
||||
if ("id_orchestrator" in msg["data"] and msg["data"]["id_orchestrator"] == DEVICE_ID):
|
||||
# Do not answer to messages coming from me
|
||||
continue
|
||||
microwave_id = msg["data"]["id_microwave"]
|
||||
print(f"\n[Main Loop] MQTT : Hello reçu de {microwave_id}.")
|
||||
# Responds
|
||||
mqtt_client.publish(config.MQTT_TOPIC_HELLO, payloads.mqtt_hello_ack(DEVICE_ID, microwave_id), qos=config.MQTT_QOS)
|
||||
print(f"[Main Loop] MQTT : Réponse Hello envoyée à {microwave_id}.")
|
||||
# TODO : Save in database
|
||||
|
||||
|
||||
print(f"\n[Main Loop] MQTT : Données traitées : {msg['data']}")
|
||||
except queue.Empty:
|
||||
pass
|
||||
|
||||
# DEBUG : Read sensors
|
||||
sensor_values = read_sensors()
|
||||
if sensor_values:
|
||||
sensor_values_print = sensor_values.copy()
|
||||
if "camera_image" in sensor_values_print:
|
||||
sensor_values_print["camera_image"] = f"<{len(sensor_values_print['camera_image'])} bytes>"
|
||||
print(f"\nCapteurs Données lues : {sensor_values_print}")
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
# Publish debug telemetry message
|
||||
print("[Main Loop] Envoi d'un message de debug sur MQTT...")
|
||||
response = mqtt_client.publish(
|
||||
config.MQTT_TOPIC_COOKING,
|
||||
f"Orchestrateur actif, ID: {DEVICE_ID}",
|
||||
qos=config.MQTT_QOS
|
||||
)
|
||||
|
||||
# This will now unblock instantly because loop_start() handles the delivery confirmation!
|
||||
response.wait_for_publish()
|
||||
print("[Main Loop] Message de debug publié avec succès.")
|
||||
|
||||
time.sleep(9)
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
except KeyboardInterrupt:
|
||||
break
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
time.sleep(1) # Prevents rapid error logging in case of persistent issues
|
||||
|
||||
# Clean termination
|
||||
if hasattr(mqtt_client._client, "loop_stop"):
|
||||
mqtt_client._client.loop_stop()
|
||||
mqtt_client.close()
|
||||
mqtt_client.close()
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
paho-mqtt>=1.6,<3
|
||||
pyserial>=3.5,<4
|
||||
# picamera2>=0.3.36,<4 # → Installed with apt install python3-picamera2
|
||||
# OpenCV
|
||||
# sudo apt install -y python3-opencv
|
||||
# sudo apt install -y opencv-data
|
||||
@@ -0,0 +1,7 @@
|
||||
# import grovepi
|
||||
|
||||
import sensors.ultrasonicRanger as ultrasonicRanger
|
||||
import sensors.temp_hum as temp_hum
|
||||
import sensors.button as button
|
||||
import sensors.gps as gps
|
||||
import sensors.camera as camera
|
||||
@@ -0,0 +1,44 @@
|
||||
import grovepi
|
||||
import time
|
||||
import threading
|
||||
from sensors.lock import grove_lock
|
||||
from shared.logging import log
|
||||
|
||||
button = 2
|
||||
button_switch_state = 0
|
||||
grovepi.pinMode(button, "INPUT")
|
||||
|
||||
button_callback = None
|
||||
|
||||
def read_button_state():
|
||||
if not grove_lock.acquire(timeout=0.05):
|
||||
return None
|
||||
try:
|
||||
return grovepi.digitalRead(button)
|
||||
except Exception as e:
|
||||
log(f"BTN Error: {e}")
|
||||
return None
|
||||
finally:
|
||||
grove_lock.release()
|
||||
|
||||
def monitor_button():
|
||||
global button_switch_state
|
||||
last_button_state = button_switch_state
|
||||
|
||||
while True:
|
||||
time.sleep(0.04)
|
||||
|
||||
current_state = read_button_state()
|
||||
|
||||
if current_state is not None:
|
||||
if current_state == 1 and last_button_state == 0:
|
||||
if button_callback:
|
||||
button_callback()
|
||||
last_button_state = current_state
|
||||
|
||||
def start_button_monitoring_thread():
|
||||
threading.Thread(target=monitor_button, daemon=True).start()
|
||||
|
||||
def set_callback(callback):
|
||||
global button_callback
|
||||
button_callback = callback
|
||||
@@ -0,0 +1,33 @@
|
||||
import grovepi
|
||||
import math
|
||||
from sensors.lock import grove_lock
|
||||
from picamera2 import Picamera2, Preview
|
||||
import time
|
||||
|
||||
picam2 = Picamera2()
|
||||
|
||||
camera_config = picam2.create_still_configuration()
|
||||
picam2.configure(camera_config)
|
||||
|
||||
picam2.start()
|
||||
time.sleep(2)
|
||||
|
||||
def preview_camera():
|
||||
picam2.start_preview(Preview.DRM)
|
||||
|
||||
def stop_preview_camera():
|
||||
picam2.stop_preview()
|
||||
|
||||
def take_picture():
|
||||
"""Takes a picture and saves it to the file system"""
|
||||
picam2.capture_file("test.jpg")
|
||||
return "test.jpg"
|
||||
|
||||
def get_picture():
|
||||
"""Returns the image bytes as base64
|
||||
"""
|
||||
file_path = take_picture()
|
||||
with open(file_path, "rb") as f:
|
||||
image_bytes = f.read()
|
||||
return image_bytes
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
import serial
|
||||
import time
|
||||
import threading
|
||||
from shared.logging import log
|
||||
from sensors.lock import serial_lock
|
||||
|
||||
def calculate_nmea_checksum(line: str) -> bool:
|
||||
"""Validates standard NMEA 0183 sentence checksum ($...*HH)."""
|
||||
if not line.startswith('$') or '*' not in line:
|
||||
return False
|
||||
|
||||
try:
|
||||
content, checksum_str = line[1:].split('*', 1)
|
||||
calculated_checksum = 0
|
||||
for char in content:
|
||||
calculated_checksum ^= ord(char)
|
||||
|
||||
return calculated_checksum == int(checksum_str[:2], 16)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
class GROVEGPS:
|
||||
def __init__(self, port='/dev/ttyAMA0', baud=9600, timeout=1):
|
||||
self.ser = serial.Serial(port, baud, timeout=timeout)
|
||||
self.clean_data()
|
||||
|
||||
def clean_data(self):
|
||||
self.timestamp = ""
|
||||
self.quality = 0
|
||||
self.satellites = 0
|
||||
self.altitude = -1.0
|
||||
self.latitude = -1.0
|
||||
self.longitude = -1.0
|
||||
|
||||
def read(self):
|
||||
"""Reads the latest GGA sentence from serial, thread-safely."""
|
||||
with serial_lock:
|
||||
# 1. Flush accumulated stale data in the UART buffer
|
||||
if self.ser.in_waiting > 0:
|
||||
self.ser.reset_input_buffer()
|
||||
|
||||
# 2. Try reading up to 15 lines to catch the freshest GGA sentence
|
||||
for _ in range(5):
|
||||
raw_bytes = self.ser.readline()
|
||||
try:
|
||||
line = raw_bytes.decode('utf-8', errors='ignore').strip()
|
||||
# log(f"GPS: Read line: {line}")
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Supports both $GPGGA and modern $GNGGA sentences
|
||||
if (line.startswith('$GPGGA') or line.startswith('$GNGGA')) and calculate_nmea_checksum(line):
|
||||
if self.parse_gga(line):
|
||||
return True
|
||||
return False
|
||||
|
||||
def parse_gga(self, line):
|
||||
self.clean_data()
|
||||
gga = line.split(',')
|
||||
|
||||
if len(gga) < 10:
|
||||
return False
|
||||
|
||||
try:
|
||||
self.timestamp = gga[1]
|
||||
self.quality = int(gga[6]) if gga[6] != "" else 0
|
||||
self.satellites = int(gga[7]) if gga[7] != "" else 0
|
||||
|
||||
# If quality > 0 and coordinates exist, convert NMEA DDDMM.MMMM to decimal degrees
|
||||
if self.quality > 0 and gga[2] != "" and gga[4] != "":
|
||||
lat_raw = float(gga[2])
|
||||
ns = gga[3]
|
||||
lon_raw = float(gga[4])
|
||||
ew = gga[5]
|
||||
|
||||
# Latitude calculation
|
||||
lat_deg = lat_raw // 100
|
||||
lat_min = lat_raw % 100
|
||||
self.latitude = lat_deg + (lat_min / 60.0)
|
||||
if ns == 'S':
|
||||
self.latitude = -self.latitude
|
||||
|
||||
# Longitude calculation
|
||||
lon_deg = lon_raw // 100
|
||||
lon_min = lon_raw % 100
|
||||
self.longitude = lon_deg + (lon_min / 60.0)
|
||||
if ew == 'W':
|
||||
self.longitude = -self.longitude
|
||||
|
||||
self.altitude = float(gga[9]) if gga[9] != "" else -1.0
|
||||
return True
|
||||
else:
|
||||
# No lock on this line
|
||||
return True
|
||||
|
||||
except (ValueError, IndexError):
|
||||
return False
|
||||
|
||||
|
||||
# Shared instance
|
||||
gps = GROVEGPS()
|
||||
|
||||
def get_gps_data():
|
||||
"""Returns GPS dictionary if fix is valid, otherwise returns None."""
|
||||
has_data = gps.read()
|
||||
|
||||
# Strictly check that we have a valid GPS lock (quality > 0 and valid coordinates)
|
||||
if has_data and gps.quality > 0 and gps.latitude != -1.0:
|
||||
return {
|
||||
"timestamp": gps.timestamp,
|
||||
"latitude": round(gps.latitude, 6),
|
||||
"longitude": round(gps.longitude, 6),
|
||||
"altitude": gps.altitude,
|
||||
"quality": gps.quality,
|
||||
"satellites": gps.satellites
|
||||
}
|
||||
else:
|
||||
log(f"GPS: No valid fix or data available. Satellites: {gps.satellites}, Quality: {gps.quality}")
|
||||
|
||||
# Return None so main.py doesn't process or log empty GPS data
|
||||
return None
|
||||
@@ -0,0 +1,2 @@
|
||||
# import orchestrateur.sensors.lib.grovepi_old as grovepi_old
|
||||
# import sensors.lib.grove_i2c_temp_hum_mini as grove_i2c_temp_hum_mini
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# GrovePi Library for using the Grove - Temperature&Humidity Sensor (http://www.seeedstudio.com/depot/Grove-TemperatureHumidity-Sensor-HighAccuracy-Mini-p-1921.html)
|
||||
#
|
||||
# The GrovePi connects the Raspberry Pi and Grove sensors. You can learn more about GrovePi here: http://www.dexterindustries.com/GrovePi
|
||||
#
|
||||
# Have a question about this library? Ask on the forums here: http://forum.dexterindustries.com/c/grovepi
|
||||
#
|
||||
# Released under the MIT license (http://choosealicense.com/licenses/mit/).
|
||||
# For more information see https://github.com/DexterInd/GrovePi/blob/master/LICENSE
|
||||
#################################################################################################################################################
|
||||
# NOTE:
|
||||
# The software for this sensor is still in development and might make your GrovePi unuable as long as this sensor is connected with the GrovePi
|
||||
#################################################################################################################################################
|
||||
import time,sys
|
||||
import RPi.GPIO as GPIO
|
||||
import smbus
|
||||
|
||||
debug = 0
|
||||
# use the bus that matches your raspi version
|
||||
rev = GPIO.RPI_REVISION
|
||||
if rev == 2 or rev == 3:
|
||||
bus = smbus.SMBus(1)
|
||||
else:
|
||||
bus = smbus.SMBus(0)
|
||||
|
||||
class th02:
|
||||
|
||||
ADDRESS = 0x40
|
||||
|
||||
TH02_REG_STATUS = 0x00
|
||||
TH02_REG_DATA_H = 0x01
|
||||
TH02_REG_DATA_L = 0x02
|
||||
TH02_REG_CONFIG = 0x03
|
||||
TH02_REG_ID = 0x11
|
||||
|
||||
TH02_STATUS_RDY_MASK = 0x01
|
||||
|
||||
TH02_CMD_MEASURE_HUMI = [0x01]
|
||||
TH02_CMD_MEASURE_TEMP = [0x11]
|
||||
|
||||
SUCCESS = 0
|
||||
|
||||
def getTemperature(self):
|
||||
bus.write_i2c_block_data(self.ADDRESS, self.TH02_REG_CONFIG, self.TH02_CMD_MEASURE_TEMP)
|
||||
|
||||
while 1:
|
||||
status=self.getStatus()
|
||||
if debug:
|
||||
print("st:",status)
|
||||
if status:
|
||||
break
|
||||
t_raw=bus.read_i2c_block_data(self.ADDRESS, self.TH02_REG_DATA_H,3)
|
||||
if debug:
|
||||
print(t_raw)
|
||||
temperature = (t_raw[1]<<8|t_raw[2])>>2
|
||||
return (temperature/32.0)-50.0
|
||||
|
||||
def getHumidity(self):
|
||||
bus.write_i2c_block_data(self.ADDRESS, self.TH02_REG_CONFIG, self.TH02_CMD_MEASURE_HUMI)
|
||||
|
||||
while 1:
|
||||
status=self.getStatus()
|
||||
if debug:
|
||||
print("st:",status)
|
||||
if status:
|
||||
break
|
||||
t_raw=bus.read_i2c_block_data(self.ADDRESS, self.TH02_REG_DATA_H,3)
|
||||
if debug:
|
||||
print(t_raw)
|
||||
temperature = (t_raw[1]<<8|t_raw[2])>>4
|
||||
return (temperature/16.0)-24.0
|
||||
|
||||
def getStatus(self):
|
||||
status=bus.read_i2c_block_data(self.ADDRESS, self.TH02_REG_STATUS,1)
|
||||
if debug:
|
||||
print(status)
|
||||
if status[0] & self.TH02_STATUS_RDY_MASK != 1:
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
t= th02()
|
||||
while True:
|
||||
print(t.getTemperature(),t.getHumidity())
|
||||
time.sleep(.5)
|
||||
@@ -0,0 +1,691 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# GrovePi Python library
|
||||
# v1.4
|
||||
#
|
||||
# This file provides the basic functions for using the GrovePi
|
||||
#
|
||||
# The GrovePi connects the Raspberry Pi and Grove sensors. You can learn more about GrovePi here: http://www.dexterindustries.com/GrovePi
|
||||
#
|
||||
# Have a question about this example? Ask on the forums here: http://forum.dexterindustries.com/c/grovepi
|
||||
#
|
||||
'''
|
||||
## License
|
||||
|
||||
The MIT License (MIT)
|
||||
|
||||
GrovePi for the Raspberry Pi: an open source platform for connecting Grove Sensors to the Raspberry Pi.
|
||||
Copyright (C) 2017 Dexter Industries
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
'''
|
||||
# Initial Date: 13 Feb 2014
|
||||
# Last Updated: 11 Nov 2016
|
||||
# http://www.dexterindustries.com/
|
||||
# Author Date Comments
|
||||
# Karan 13 Feb 2014 Initial Authoring
|
||||
# 11 Nov 2016 I2C retries added for faster IO
|
||||
# DHT function updated to look for nan's
|
||||
|
||||
__version__ = '1.4.1'
|
||||
|
||||
import sys
|
||||
import time
|
||||
import math
|
||||
import struct
|
||||
import numpy
|
||||
|
||||
import di_i2c
|
||||
|
||||
def set_bus(bus):
|
||||
global i2c
|
||||
i2c = di_i2c.DI_I2C(bus = bus, address = address)
|
||||
|
||||
address = 0x04
|
||||
max_recv_size = 10
|
||||
set_bus("RPI_1SW")
|
||||
|
||||
if sys.version_info<(3,0):
|
||||
p_version = 2
|
||||
else:
|
||||
p_version = 3
|
||||
|
||||
# Earliest version of the firmware to work with
|
||||
works_with_firmware = [
|
||||
"1.4.0"
|
||||
]
|
||||
|
||||
# interrupt operations
|
||||
COUNT_CHANGES = 0
|
||||
COUNT_LOW_DURATION = 1
|
||||
|
||||
# interrupt trigger mode
|
||||
CHANGE = 1
|
||||
FALLING = 2
|
||||
RISING = 3
|
||||
|
||||
# This allows us to be more specific about which commands contain unused bytes
|
||||
unused = 0
|
||||
retries = 10
|
||||
additional_waiting = 0
|
||||
|
||||
# Get firmware version
|
||||
version_cmd = [8]
|
||||
# No data is available from the GrovePi
|
||||
data_not_available_cmd = [23]
|
||||
|
||||
# Command Format
|
||||
# digitalRead() command format header
|
||||
dRead_cmd = [1]
|
||||
# digitalWrite() command format header
|
||||
dWrite_cmd = [2]
|
||||
# analogRead() command format header
|
||||
aRead_cmd = [3]
|
||||
# analogWrite() command format header
|
||||
aWrite_cmd = [4]
|
||||
# pinMode() command format header
|
||||
pMode_cmd = [5]
|
||||
# Ultrasonic read
|
||||
uRead_cmd = [7]
|
||||
# Accelerometer (+/- 1.5g) read
|
||||
acc_xyz_cmd = [20]
|
||||
# RTC get time
|
||||
rtc_getTime_cmd = [30]
|
||||
# DHT Pro sensor temperature
|
||||
dht_temp_cmd = [40]
|
||||
|
||||
# Grove LED Bar commands
|
||||
# Initialise
|
||||
ledBarInit_cmd = [50]
|
||||
# Set orientation
|
||||
ledBarOrient_cmd = [51]
|
||||
# Set level
|
||||
ledBarLevel_cmd = [52]
|
||||
# Set single LED
|
||||
ledBarSetOne_cmd = [53]
|
||||
# Toggle single LED
|
||||
ledBarToggleOne_cmd = [54]
|
||||
# Set all LEDs
|
||||
ledBarSet_cmd = [55]
|
||||
# Get current state
|
||||
ledBarGet_cmd = [56]
|
||||
|
||||
# Grove 4 Digit Display commands
|
||||
# Initialise
|
||||
fourDigitInit_cmd = [70]
|
||||
# Set brightness, not visible until next cmd
|
||||
fourDigitBrightness_cmd = [71]
|
||||
# Set numeric value without leading zeros
|
||||
fourDigitValue_cmd = [72]
|
||||
# Set numeric value with leading zeros
|
||||
fourDigitValueZeros_cmd = [73]
|
||||
# Set individual digit
|
||||
fourDigitIndividualDigit_cmd = [74]
|
||||
# Set individual leds of a segment
|
||||
fourDigitIndividualLeds_cmd = [75]
|
||||
# Set left and right values with colon
|
||||
fourDigitScore_cmd = [76]
|
||||
# Analog read for n seconds
|
||||
fourDigitAnalogRead_cmd = [77]
|
||||
# Entire display on
|
||||
fourDigitAllOn_cmd = [78]
|
||||
# Entire display off
|
||||
fourDigitAllOff_cmd = [79]
|
||||
|
||||
# Grove Chainable RGB LED commands
|
||||
# Store color for later use
|
||||
storeColor_cmd = [90]
|
||||
# Initialise
|
||||
chainableRgbLedInit_cmd = [91]
|
||||
# Initialise and test with a simple color
|
||||
chainableRgbLedTest_cmd = [92]
|
||||
# Set one or more leds to the stored color by pattern
|
||||
chainableRgbLedSetPattern_cmd = [93]
|
||||
# set one or more leds to the stored color by modulo
|
||||
chainableRgbLedSetModulo_cmd = [94]
|
||||
# sets leds similar to a bar graph, reversible
|
||||
chainableRgbLedSetLevel_cmd = [95]
|
||||
|
||||
# Read the button from IR sensor
|
||||
ir_read_cmd = [21]
|
||||
# Set pin for the IR receiver
|
||||
ir_recv_pin_cmd = [22]
|
||||
# Check if there's data coming from the IR receiver
|
||||
ir_read_isdata = [24]
|
||||
|
||||
# Interrupt-based devices
|
||||
isr_set_cmd = [6]
|
||||
isr_unset_cmd = [9]
|
||||
isr_read_cmd = [10]
|
||||
isr_clear_cmd = [11]
|
||||
isr_active_cmd = [12]
|
||||
|
||||
# Grove Encoders
|
||||
encoder_read_cmd = [13]
|
||||
encoder_en_cmd = [14]
|
||||
encoder_dis_cmd = [15]
|
||||
|
||||
# Dust, Encoder & Flow Sensor commands
|
||||
# dust_sensor_read_cmd=[10]
|
||||
# dust_sensor_en_cmd=[14]
|
||||
# dust_sensor_dis_cmd=[15]
|
||||
# dust_sensor_int_cmd=[9]
|
||||
# dust_sensor_read_int_cmd=[6]
|
||||
# flow_read_cmd=[12]
|
||||
# flow_disable_cmd=[13]
|
||||
# flow_en_cmd=[18]
|
||||
|
||||
|
||||
# Function declarations of the various functions used for encoding and sending
|
||||
# data from RPi to Arduino
|
||||
|
||||
# Write I2C block to the GrovePi
|
||||
def write_i2c_block(block, custom_timing = None):
|
||||
'''
|
||||
Now catches and raises Keyboard Interrupt that the user is responsible to catch.
|
||||
'''
|
||||
counter = 0
|
||||
reg = block[0]
|
||||
data = block[1:]
|
||||
while counter < 3:
|
||||
try:
|
||||
i2c.write_reg_list(reg, data)
|
||||
time.sleep(0.002 + additional_waiting)
|
||||
return
|
||||
except KeyboardInterrupt:
|
||||
raise KeyboardInterrupt
|
||||
except:
|
||||
counter += 1
|
||||
time.sleep(0.003)
|
||||
continue
|
||||
|
||||
# Read I2C block from the GrovePi
|
||||
def read_i2c_block(no_bytes = max_recv_size):
|
||||
'''
|
||||
Now catches and raises Keyboard Interrupt that the user is responsible to catch.
|
||||
'''
|
||||
data = data_not_available_cmd
|
||||
counter = 0
|
||||
while data[0] in [data_not_available_cmd[0], 255] and counter < 3:
|
||||
try:
|
||||
data = i2c.read_list(reg = None, len = no_bytes)
|
||||
time.sleep(0.002 + additional_waiting)
|
||||
if counter > 0:
|
||||
counter = 0
|
||||
except KeyboardInterrupt:
|
||||
raise KeyboardInterrupt
|
||||
except:
|
||||
counter += 1
|
||||
time.sleep(0.003)
|
||||
|
||||
return data
|
||||
|
||||
def read_identified_i2c_block(read_command_id, no_bytes):
|
||||
data = [-1]
|
||||
while len(data) <= 1:
|
||||
data = read_i2c_block(no_bytes + 1)
|
||||
|
||||
return data[1:]
|
||||
|
||||
# Arduino Digital Read
|
||||
def digitalRead(pin):
|
||||
write_i2c_block(dRead_cmd + [pin, unused, unused])
|
||||
data = read_identified_i2c_block( dRead_cmd, no_bytes = 1)[0]
|
||||
return data
|
||||
|
||||
# Arduino Digital Write
|
||||
def digitalWrite(pin, value):
|
||||
write_i2c_block(dWrite_cmd + [pin, value, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Read analog value from Pin
|
||||
def analogRead(pin):
|
||||
write_i2c_block(aRead_cmd + [pin, unused, unused])
|
||||
number = read_identified_i2c_block(aRead_cmd, no_bytes = 2)
|
||||
return number[0] * 256 + number[1]
|
||||
|
||||
|
||||
# Write PWM
|
||||
def analogWrite(pin, value):
|
||||
write_i2c_block(aWrite_cmd + [pin, value, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Setting Up Pin mode on Arduino
|
||||
def pinMode(pin, mode):
|
||||
if mode == "OUTPUT":
|
||||
write_i2c_block(pMode_cmd + [pin, 1, unused])
|
||||
elif mode == "INPUT":
|
||||
write_i2c_block(pMode_cmd + [pin, 0, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
|
||||
# Read temp in Celsius from Grove Temperature Sensor
|
||||
def temp(pin, model = '1.0'):
|
||||
# each of the sensor revisions use different thermistors, each with their own B value constant
|
||||
if model == '1.2':
|
||||
bValue = 4250 # sensor v1.2 uses thermistor ??? (assuming NCP18WF104F03RC until SeeedStudio clarifies)
|
||||
elif model == '1.1':
|
||||
bValue = 4250 # sensor v1.1 uses thermistor NCP18WF104F03RC
|
||||
else:
|
||||
bValue = 3975 # sensor v1.0 uses thermistor TTC3A103*39H
|
||||
a = analogRead(pin)
|
||||
resistance = (float)(1023 - a) * 10000 / a
|
||||
t = (float)(1 / (math.log(resistance / 10000) / bValue + 1 / 298.15) - 273.15)
|
||||
return t
|
||||
|
||||
|
||||
# Read value from Grove Ultrasonic
|
||||
def ultrasonicRead(pin):
|
||||
write_i2c_block(uRead_cmd + [pin, unused, unused])
|
||||
number = read_identified_i2c_block(uRead_cmd, no_bytes = 2)
|
||||
return (number[0] * 256 + number[1])
|
||||
|
||||
|
||||
# Read the firmware version
|
||||
def version():
|
||||
write_i2c_block(version_cmd + [unused, unused, unused])
|
||||
number = read_identified_i2c_block(version_cmd, no_bytes = 3)
|
||||
return "%s.%s.%s" % (number[0], number[1], number[2])
|
||||
|
||||
|
||||
# Read Grove Accelerometer (+/- 1.5g) XYZ value
|
||||
# Need to investigate why this reports what was read with the previous command
|
||||
# Doesn't look to be implemented on the GrovePi
|
||||
def acc_xyz():
|
||||
write_i2c_block(acc_xyz_cmd + [unused, unused, unused])
|
||||
number = read_identified_i2c_block(acc_xyz_cmd, no_bytes = 3)
|
||||
if number[1] > 32:
|
||||
number[1] = - (number[1] - 224)
|
||||
if number[2] > 32:
|
||||
number[2] = - (number[2] - 224)
|
||||
if number[3] > 32:
|
||||
number[3] = - (number[3] - 224)
|
||||
return (number[0], number[1], number[2])
|
||||
|
||||
|
||||
# Read from Grove RTC
|
||||
# Doesn't look to be implemented on the GrovePi
|
||||
def rtc_getTime():
|
||||
write_i2c_block(rtc_getTime_cmd + [unused, unused, unused])
|
||||
number = read_i2c_block()
|
||||
return number
|
||||
|
||||
# Read and return temperature and humidity from Grove DHT Pro
|
||||
def dht(pin, module_type):
|
||||
write_i2c_block(dht_temp_cmd + [pin, module_type, unused])
|
||||
number = read_identified_i2c_block(dht_temp_cmd, no_bytes = 8)
|
||||
|
||||
if p_version==2:
|
||||
h=''
|
||||
for element in (number[0:4]):
|
||||
h+=chr(element)
|
||||
|
||||
t_val=struct.unpack('f', h)
|
||||
t = round(t_val[0], 2)
|
||||
|
||||
h = ''
|
||||
for element in (number[4:8]):
|
||||
h+=chr(element)
|
||||
|
||||
hum_val=struct.unpack('f',h)
|
||||
hum = round(hum_val[0], 2)
|
||||
else:
|
||||
t_val=bytearray(number[0:4])
|
||||
h_val=bytearray(number[4:8])
|
||||
t=round(struct.unpack('f',t_val)[0],2)
|
||||
hum=round(struct.unpack('f',h_val)[0],2)
|
||||
if t > -100.0 and t <150.0 and hum >= 0.0 and hum<=100.0:
|
||||
return [t, hum]
|
||||
else:
|
||||
return [float('nan'),float('nan')]
|
||||
|
||||
# Grove - Infrared Receiver - get the commands received from the Grove IR sensor
|
||||
def ir_read_signal():
|
||||
write_i2c_block(ir_read_cmd + [unused, unused, unused])
|
||||
data_back = read_identified_i2c_block(ir_read_cmd, no_bytes = 7)
|
||||
|
||||
return (data_back[0],
|
||||
data_back[1] + data_back[2] * 256,
|
||||
data_back[3] + data_back[4] * 256 + data_back[5] * (256 ** 2) + data_back[6] * (256 ** 3))
|
||||
|
||||
# Grove - Infrared Receiver - set the pin on which the Grove IR sensor is connected
|
||||
def ir_recv_pin(pin):
|
||||
write_i2c_block(ir_recv_pin_cmd + [pin, unused, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
|
||||
# Grove - Infrared Receiver - check if there's any data that hasn't been read so far
|
||||
def ir_is_data():
|
||||
write_i2c_block(ir_read_isdata + 3 * [unused])
|
||||
number = read_identified_i2c_block(ir_read_isdata, no_bytes = 1)
|
||||
|
||||
return number[0] != 0
|
||||
|
||||
# after a list of numerical values is provided
|
||||
# the function returns a list with the outlier(or extreme) values removed
|
||||
# make the std_factor_threshold bigger so that filtering becomes less strict
|
||||
# and make the std_factor_threshold smaller to get the opposite
|
||||
def statisticalNoiseReduction(values, std_factor_threshold = 2):
|
||||
if len(values) == 0:
|
||||
return []
|
||||
|
||||
mean = numpy.mean(values)
|
||||
standard_deviation = numpy.std(values)
|
||||
|
||||
if standard_deviation == 0:
|
||||
return values
|
||||
|
||||
filtered_values = [element for element in values if element > mean - std_factor_threshold * standard_deviation]
|
||||
filtered_values = [element for element in filtered_values if element < mean + std_factor_threshold * standard_deviation]
|
||||
|
||||
return filtered_values
|
||||
|
||||
|
||||
# Grove LED Bar - initialise
|
||||
# orientation: (0 = red to green, 1 = green to red)
|
||||
def ledBar_init(pin, orientation):
|
||||
write_i2c_block(ledBarInit_cmd + [pin, orientation, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove LED Bar - set orientation
|
||||
# orientation: (0 = red to green, 1 = green to red)
|
||||
def ledBar_orientation(pin, orientation):
|
||||
write_i2c_block(ledBarOrient_cmd + [pin, orientation, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove LED Bar - set level
|
||||
# level: (0-10)
|
||||
def ledBar_setLevel(pin, level):
|
||||
write_i2c_block(ledBarLevel_cmd + [pin, level, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove LED Bar - set single led
|
||||
# led: which led (1-10)
|
||||
# state: off or on (0-1)
|
||||
def ledBar_setLed(pin, led, state):
|
||||
write_i2c_block(ledBarSetOne_cmd + [pin, led, state])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove LED Bar - toggle single led
|
||||
# led: which led (1-10)
|
||||
def ledBar_toggleLed(pin, led):
|
||||
write_i2c_block(ledBarToggleOne_cmd + [pin, led, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove LED Bar - set all leds
|
||||
# state: (0-1023) or (0x00-0x3FF) or (0b0000000000-0b1111111111) or (int('0000000000',2)-int('1111111111',2))
|
||||
def ledBar_setBits(pin, state):
|
||||
byte1 = state & 255
|
||||
byte2 = state >> 8
|
||||
write_i2c_block(ledBarSet_cmd + [pin, byte1, byte2])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove LED Bar - get current state
|
||||
# state: (0-1023) a bit for each of the 10 LEDs
|
||||
def ledBar_getBits(pin):
|
||||
write_i2c_block(ledBarGet_cmd + [pin, unused, unused])
|
||||
block = read_identified_i2c_block(ledBarGet_cmd, no_bytes = 2)
|
||||
return block[0] ^ (block[1] << 8)
|
||||
|
||||
|
||||
# Grove 4 Digit Display - initialise
|
||||
def fourDigit_init(pin):
|
||||
write_i2c_block(fourDigitInit_cmd + [pin, unused, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove 4 Digit Display - set numeric value with or without leading zeros
|
||||
# value: (0-65535) or (0000-FFFF)
|
||||
def fourDigit_number(pin, value, leading_zero):
|
||||
# split the value into two bytes so we can render 0000-FFFF on the display
|
||||
byte1 = value & 255
|
||||
byte2 = value >> 8
|
||||
# separate commands to overcome current 4 bytes per command limitation
|
||||
if (leading_zero):
|
||||
write_i2c_block(fourDigitValue_cmd + [pin, byte1, byte2])
|
||||
else:
|
||||
write_i2c_block(fourDigitValueZeros_cmd + [pin, byte1, byte2])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove 4 Digit Display - set brightness
|
||||
# brightness: (0-7)
|
||||
def fourDigit_brightness(pin, brightness):
|
||||
# not actually visible until next command is executed
|
||||
write_i2c_block(fourDigitBrightness_cmd + [pin, brightness, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove 4 Digit Display - set individual segment (0-9,A-F)
|
||||
# segment: (0-3)
|
||||
# value: (0-15) or (0-F)
|
||||
def fourDigit_digit(pin, segment, value):
|
||||
write_i2c_block(fourDigitIndividualDigit_cmd + [pin, segment, value])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove 4 Digit Display - set 7 individual leds of a segment
|
||||
# segment: (0-3)
|
||||
# leds: (0-255) or (0-0xFF) one bit per led, segment 2 is special, 8th bit is the colon
|
||||
def fourDigit_segment(pin, segment, leds):
|
||||
write_i2c_block(fourDigitIndividualLeds_cmd + [pin, segment, leds])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove 4 Digit Display - set left and right values (0-99), with leading zeros and a colon
|
||||
# left: (0-255) or (0-FF)
|
||||
# right: (0-255) or (0-FF)
|
||||
# colon will be lit
|
||||
def fourDigit_score(pin, left, right):
|
||||
write_i2c_block(fourDigitScore_cmd + [pin, left, right])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove 4 Digit Display - display analogRead value for n seconds, 4 samples per second
|
||||
# analog: analog pin to read
|
||||
# duration: analog read for this many seconds
|
||||
def fourDigit_monitor(pin, analog, duration):
|
||||
write_i2c_block(fourDigitAnalogRead_cmd + [pin, analog, duration])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
time.sleep(duration)
|
||||
return 1
|
||||
|
||||
# Grove 4 Digit Display - turn entire display on (88:88)
|
||||
def fourDigit_on(pin):
|
||||
write_i2c_block(fourDigitAllOn_cmd + [pin, unused, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove 4 Digit Display - turn entire display off
|
||||
def fourDigit_off(pin):
|
||||
write_i2c_block(fourDigitAllOff_cmd + [pin, unused, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove Chainable RGB LED - store a color for later use
|
||||
# red: 0-255
|
||||
# green: 0-255
|
||||
# blue: 0-255
|
||||
def storeColor(red, green, blue):
|
||||
write_i2c_block(storeColor_cmd + [red, green, blue])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove Chainable RGB LED - initialise
|
||||
# numLeds: how many leds do you have in the chain
|
||||
def chainableRgbLed_init(pin, numLeds):
|
||||
write_i2c_block(chainableRgbLedInit_cmd + [pin, numLeds, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove Chainable RGB LED - initialise and test with a simple color
|
||||
# numLeds: how many leds do you have in the chain
|
||||
# testColor: (0-7) 3 bits in total - a bit for red, green and blue, eg. 0x04 == 0b100 (0bRGB) == rgb(255, 0, 0) == #FF0000 == red
|
||||
# ie. 0 black, 1 blue, 2 green, 3 cyan, 4 red, 5 magenta, 6 yellow, 7 white
|
||||
def chainableRgbLed_test(pin, numLeds, testColor):
|
||||
write_i2c_block(chainableRgbLedTest_cmd + [pin, numLeds, testColor])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove Chainable RGB LED - set one or more leds to the stored color by pattern
|
||||
# pattern: (0-3) 0 = this led only, 1 all leds except this led, 2 this led and all leds inwards, 3 this led and all leds outwards
|
||||
# whichLed: index of led you wish to set counting outwards from the GrovePi, 0 = led closest to the GrovePi
|
||||
def chainableRgbLed_pattern(pin, pattern, whichLed):
|
||||
write_i2c_block(chainableRgbLedSetPattern_cmd + [pin, pattern, whichLed])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove Chainable RGB LED - set one or more leds to the stored color by modulo
|
||||
# offset: index of led you wish to start at, 0 = led closest to the GrovePi, counting outwards
|
||||
# divisor: when 1 (default) sets stored color on all leds >= offset, when 2 sets every 2nd led >= offset and so on
|
||||
def chainableRgbLed_modulo(pin, offset, divisor):
|
||||
write_i2c_block(chainableRgbLedSetModulo_cmd + [pin, offset, divisor])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
# Grove Chainable RGB LED - sets leds similar to a bar graph, reversible
|
||||
# level: (0-10) the number of leds you wish to set to the stored color
|
||||
# reversible (0-1) when 0 counting outwards from GrovePi, 0 = led closest to the GrovePi, otherwise counting inwards
|
||||
def chainableRgbLed_setLevel(pin, level, reverse):
|
||||
write_i2c_block(chainableRgbLedSetLevel_cmd + [pin, level, reverse])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
return 1
|
||||
|
||||
def set_pin_interrupt(pin, ftype, interrupt_mode, period):
|
||||
'''
|
||||
Attach an interrupt to a pin.
|
||||
|
||||
pin - D2-D8 pins
|
||||
ftype - 0 for COUNT_CHANGES, 1 for COUNT_LOW_DURATION
|
||||
interrupt_mode - 1 for CHANGE, 2 for FALLING, 3 for RISING
|
||||
period - as measured in ms (max 65535 ms)
|
||||
'''
|
||||
period_high = period >> 8
|
||||
period_low = period & 0xff
|
||||
combined_params = (pin & 0x0f) + ((ftype & 0x03) << 4) + ((interrupt_mode & 0x03) << 6)
|
||||
write_i2c_block(isr_set_cmd + [combined_params, period_high, period_low])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
|
||||
def unset_pin_interrupt(pin):
|
||||
'''
|
||||
Detach an interrupt from a pin.
|
||||
|
||||
pin - D2-D8 pins
|
||||
'''
|
||||
write_i2c_block(isr_unset_cmd + [pin, unused, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
|
||||
def unset_all_interrupts():
|
||||
'''
|
||||
Detach all attached interrupts from all D2-D8 pins.
|
||||
|
||||
pin - D2-D8 pins
|
||||
'''
|
||||
write_i2c_block(isr_clear_cmd + 3 * [unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
|
||||
def is_interrupt_active(pin):
|
||||
write_i2c_block(isr_active_cmd + [pin, unused, unused])
|
||||
data = read_identified_i2c_block(isr_active_cmd, no_bytes = 2)
|
||||
value = data[1] >> pin
|
||||
return value != 0
|
||||
|
||||
def get_active_interrupts():
|
||||
'''
|
||||
Get list of attached interrupts for a given pin or all of them.
|
||||
|
||||
pin - D2-D8 pins; if it's 255 return the state of all pins
|
||||
'''
|
||||
pin = 255
|
||||
write_i2c_block(isr_active_cmd + [pin, unused, unused])
|
||||
data = read_identified_i2c_block(isr_active_cmd, no_bytes = 2)
|
||||
value = data[0] + (data[1] << 8)
|
||||
active_interrupts = [i for i in range(2 * 8) if ((value >> i) & 0x01)]
|
||||
return active_interrupts
|
||||
|
||||
def read_interrupt_state(pin):
|
||||
'''
|
||||
Read number of pulses/changes on given port that occurred within a time period.
|
||||
|
||||
pin - D2-D8 pins
|
||||
'''
|
||||
write_i2c_block(isr_read_cmd + [pin, unused, unused])
|
||||
data = read_identified_i2c_block(isr_read_cmd, no_bytes = 4)
|
||||
value = data[0] + (data[1] << 8) + (data[2] << 16) + (data[3] << 24)
|
||||
return value
|
||||
|
||||
def dust_sensor_en(pin = 2, period = 30000):
|
||||
set_pin_interrupt(pin, ftype=COUNT_LOW_DURATION, interrupt_mode=CHANGE, period=period)
|
||||
|
||||
def dust_sensor_dis(pin = 2):
|
||||
unset_pin_interrupt(pin)
|
||||
|
||||
def dust_sensor_read(pin = 2, period = 30000):
|
||||
'''
|
||||
By default, the sample rate is set to 1 at every 30 seconds and this
|
||||
function was written only for that interval.
|
||||
|
||||
If you wish to use a different
|
||||
interval, then use dust_sensor_read_more function. To set a
|
||||
different interval, use set_dust_sensor_interval function.
|
||||
'''
|
||||
lpo = read_interrupt_state(pin)
|
||||
percentage = 100.0 * lpo / period
|
||||
concentration = 1.1 * percentage ** 3 - 3.8 * percentage ** 2 + 520 * percentage + 0.62
|
||||
|
||||
return lpo, percentage, concentration
|
||||
|
||||
def encoder_en(pin = 2, steps = 32):
|
||||
write_i2c_block(encoder_en_cmd + [pin, steps, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
|
||||
def encoder_dis(pin = 2):
|
||||
write_i2c_block(encoder_dis_cmd + [pin, unused, unused])
|
||||
read_i2c_block(no_bytes = 1)
|
||||
|
||||
def encoderRead(pin = 2):
|
||||
write_i2c_block(encoder_read_cmd + [pin, unused, unused])
|
||||
data = read_identified_i2c_block(encoder_read_cmd, no_bytes = 4)
|
||||
value = data[0] + (data[1] << 8) + (data[2] << 16) + (data[3] << 24)
|
||||
return value
|
||||
|
||||
def flowEnable(pin = 2, period = 2000):
|
||||
set_pin_interrupt(pin, ftype=COUNT_CHANGES, interrupt_mode=RISING, period=period)
|
||||
|
||||
def flowDisable(pin = 2):
|
||||
unset_pin_interrupt(pin)
|
||||
|
||||
def flowRead(pin = 2):
|
||||
val = read_interrupt_state(pin)
|
||||
return val
|
||||
|
||||
def main():
|
||||
print("library supports this fw versions: " +
|
||||
" ".join('{}'.format(k[1]) for k in enumerate(works_with_firmware)))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
import threading
|
||||
# Dedicated lock for I2C bus access (used by GrovePi sensors)
|
||||
grove_lock = threading.Lock()
|
||||
# Dedicated lock for UART/Serial port access
|
||||
serial_lock = threading.Lock()
|
||||
@@ -0,0 +1,52 @@
|
||||
# from sensors.lib import grove_i2c_temp_hum_mini
|
||||
|
||||
# t= grove_i2c_temp_hum_mini.th02()
|
||||
|
||||
# def get_temperature():
|
||||
# """Get the temperature in Celsius from the TH02 sensor."""
|
||||
# # try:
|
||||
# return t.getTemperature()
|
||||
# # except Exception as e:
|
||||
# # print(f"Error reading temperature: {e}")
|
||||
# # return None
|
||||
|
||||
# def get_humidity():
|
||||
# """Get the humidity in percentage from the TH02 sensor."""
|
||||
# # try:
|
||||
# return t.getHumidity()
|
||||
# # except Exception as e:
|
||||
# # print(f"Error reading humidity: {e}")
|
||||
# # return None
|
||||
|
||||
# import seeed_dht
|
||||
|
||||
# sensor = seeed_dht.DHT("11", 4) # DHT11 sensor on GPIO pin 4
|
||||
|
||||
# def get_humidity_and_temperature():
|
||||
# humi, temp = sensor.read()
|
||||
# return humi, temp
|
||||
|
||||
|
||||
# import sensors.lib.grovepi as grovepi
|
||||
import grovepi
|
||||
import math
|
||||
from sensors.lock import grove_lock
|
||||
|
||||
# Connect the Grove Temperature & Humidity Sensor Pro to digital port D3
|
||||
# This example uses the blue colored sensor.
|
||||
# SIG,NC,VCC,GND
|
||||
sensor = 3 # The Sensor goes on digital port 3.
|
||||
|
||||
# temp_humidity_sensor_type
|
||||
# Grove Base Kit comes with the blue sensor.
|
||||
blue = 0 # The Blue colored sensor.
|
||||
white = 1 # The White colored sensor.
|
||||
|
||||
def get_temperature_and_humidity():
|
||||
with grove_lock:
|
||||
[temp,humidity] = grovepi.dht(sensor,blue)
|
||||
if math.isnan(temp) == False and math.isnan(humidity) == False:
|
||||
return temp, humidity
|
||||
else:
|
||||
print("Error reading from DHT sensor")
|
||||
return None, None
|
||||
@@ -0,0 +1,32 @@
|
||||
import grovepi
|
||||
from sensors.lock import grove_lock
|
||||
|
||||
# Connect the Grove Ultrasonic Ranger to digital port D4
|
||||
# SIG,NC,VCC,GND
|
||||
ULTRASONIC_RANGER_PORT = 4
|
||||
|
||||
def read_ultrasonic_ranger(ultrasonic_ranger=ULTRASONIC_RANGER_PORT):
|
||||
if not grove_lock.acquire(timeout=1.0):
|
||||
print("Ultrasonic: Lock acquisition timed out")
|
||||
return None
|
||||
|
||||
try:
|
||||
return grovepi.ultrasonicRead(ultrasonic_ranger)
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return None
|
||||
finally:
|
||||
grove_lock.release()
|
||||
|
||||
def get_dish_height():
|
||||
"""Returns the height of the dish in centimeters."""
|
||||
distance = read_ultrasonic_ranger()
|
||||
if distance is not None:
|
||||
# Assuming the ultrasonic sensor is mounted at a fixed height above the dish
|
||||
# and pointing downwards, we can calculate the height of the dish.
|
||||
# For example, if the sensor is 30 cm above the dish when it's empty:
|
||||
SENSOR_HEIGHT = 30 # cm
|
||||
dish_height = SENSOR_HEIGHT - distance
|
||||
return max(dish_height, 0) # Ensure height is not negative
|
||||
else:
|
||||
return None
|
||||
@@ -1,2 +0,0 @@
|
||||
paho-mqtt>=1.6,<3
|
||||
pyserial>=3.5,<4
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import shared.deviceTypes as deviceTypes
|
||||
import shared.config as config
|
||||
import shared.payloads as payloads
|
||||
|
||||
def get_lora(*args, **kwargs):
|
||||
from .lora_device import get_lora_device
|
||||
|
||||
+6
-3
@@ -1,12 +1,15 @@
|
||||
|
||||
DEBUG=True
|
||||
|
||||
# LoRa
|
||||
HEARTBEAT_INTERVAL = 10
|
||||
HEARTBEAT_INTERVAL = 30
|
||||
|
||||
# MQTT
|
||||
MQTT_BROKER_HOST = "192.168.50.1"
|
||||
MQTT_TOPIC_HELLO = b"smartwave/hello"
|
||||
MQTT_TOPIC_SENSOR = b"smartwave/sensor"
|
||||
MQTT_TOPIC_COOKING = b"smartwave/cooking"
|
||||
MQTT_KEEPALIVE = 30
|
||||
USE_TLS = True
|
||||
MQTT_QOS = 1
|
||||
MQTT_QOS = 1
|
||||
# Long because messages are stored into the broker and will be sent when the orchestrator is back online.
|
||||
MQTT_HELLO_INTERVAL = 30
|
||||
@@ -0,0 +1,6 @@
|
||||
from shared.config import DEBUG
|
||||
|
||||
def log(message):
|
||||
"""Log a message to the console if DEBUG is enabled."""
|
||||
if DEBUG:
|
||||
print(f"\n{message}")
|
||||
+39
-7
@@ -13,13 +13,13 @@ except ImportError:
|
||||
from umqtt.simple import MQTTClient as _MQTTClient
|
||||
BACKEND_NAME = "umqtt.simple"
|
||||
IS_MICROPYTHON = True
|
||||
except ImportError:
|
||||
try:
|
||||
from umqtt.robust import MQTTClient as _MQTTClient
|
||||
BACKEND_NAME = "umqtt.robust"
|
||||
IS_MICROPYTHON = True
|
||||
except ImportError as exc:
|
||||
raise ImportError("No MQTT client found. Expected paho.mqtt or umqtt.") from exc
|
||||
# except ImportError:
|
||||
# try:
|
||||
# from umqtt.robust import MQTTClient as _MQTTClient
|
||||
# BACKEND_NAME = "umqtt.robust"
|
||||
# IS_MICROPYTHON = True
|
||||
except ImportError as exc:
|
||||
raise ImportError("No MQTT client found. Expected paho.mqtt or umqtt.") from exc
|
||||
|
||||
|
||||
DEFAULT_PORT = 8884
|
||||
@@ -179,6 +179,38 @@ class BrokerClient:
|
||||
topic = topic.decode('utf-8')
|
||||
|
||||
return client.subscribe(topic, qos=qos)
|
||||
|
||||
def unsubscribe(self, topic):
|
||||
client = self.open()
|
||||
if IS_MICROPYTHON:
|
||||
import struct
|
||||
# Ensure the topic is bytes for writing to the socket
|
||||
topic_bytes = topic if isinstance(topic, bytes) else topic.encode('utf-8')
|
||||
|
||||
# 1. Build the MQTT unsubscribe packet header
|
||||
pkt = bytearray(b"\xa2\0\0\0")
|
||||
client.pid += 1
|
||||
|
||||
# Packet length is: 2 bytes (PID) + 2 bytes (topic length indicator) + topic string length
|
||||
struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic_bytes), client.pid)
|
||||
|
||||
# 2. Write the packet to the socket
|
||||
client.sock.write(pkt)
|
||||
client._send_str(topic_bytes)
|
||||
|
||||
# 3. Wait for the UNSUBACK confirmation frame (0xB0) from the broker
|
||||
while True:
|
||||
op = client.wait_msg()
|
||||
if op == 0xB0:
|
||||
resp = client.sock.read(3)
|
||||
assert resp[1] == pkt[2] and resp[2] == pkt[3]
|
||||
return client
|
||||
return client
|
||||
|
||||
if isinstance(topic, bytes):
|
||||
topic = topic.decode('utf-8')
|
||||
|
||||
return client.unsubscribe(topic)
|
||||
|
||||
def _on_micropython_message(self, topic, payload):
|
||||
self._store_message(topic, payload, None, False)
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
try:
|
||||
import ujson as json
|
||||
except ImportError:
|
||||
import json
|
||||
|
||||
def as_json(data):
|
||||
"""Convert a dictionary to a JSON string."""
|
||||
try:
|
||||
return json.dumps(data)
|
||||
except Exception as e:
|
||||
print("[Payloads] Error converting to JSON:", e)
|
||||
return "{}" # Return an empty JSON object on error
|
||||
|
||||
def mqtt_hello(id_microwave):
|
||||
return as_json({
|
||||
"id_microwave": id_microwave
|
||||
})
|
||||
|
||||
def mqtt_hello_ack(id_orchestrator, id_microwave):
|
||||
return as_json({
|
||||
"id_microwave": id_microwave,
|
||||
"id_orchestrator": id_orchestrator
|
||||
})
|
||||
@@ -1 +0,0 @@
|
||||
pip
|
||||
@@ -1,163 +0,0 @@
|
||||
Metadata-Version: 2.4
|
||||
Name: urllib3
|
||||
Version: 2.7.0
|
||||
Summary: HTTP library with thread-safe connection pooling, file post, and more.
|
||||
Project-URL: Changelog, https://github.com/urllib3/urllib3/blob/main/CHANGES.rst
|
||||
Project-URL: Documentation, https://urllib3.readthedocs.io
|
||||
Project-URL: Code, https://github.com/urllib3/urllib3
|
||||
Project-URL: Issue tracker, https://github.com/urllib3/urllib3/issues
|
||||
Author-email: Andrey Petrov <andrey.petrov@shazow.net>
|
||||
Maintainer-email: Seth Michael Larson <sethmichaellarson@gmail.com>, Quentin Pradet <quentin@pradet.me>, Illia Volochii <illia.volochii@gmail.com>
|
||||
License-Expression: MIT
|
||||
License-File: LICENSE.txt
|
||||
Keywords: filepost,http,httplib,https,pooling,ssl,threadsafe,urllib
|
||||
Classifier: Environment :: Web Environment
|
||||
Classifier: Intended Audience :: Developers
|
||||
Classifier: Operating System :: OS Independent
|
||||
Classifier: Programming Language :: Python
|
||||
Classifier: Programming Language :: Python :: 3
|
||||
Classifier: Programming Language :: Python :: 3 :: Only
|
||||
Classifier: Programming Language :: Python :: 3.10
|
||||
Classifier: Programming Language :: Python :: 3.11
|
||||
Classifier: Programming Language :: Python :: 3.12
|
||||
Classifier: Programming Language :: Python :: 3.13
|
||||
Classifier: Programming Language :: Python :: 3.14
|
||||
Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
|
||||
Classifier: Programming Language :: Python :: Implementation :: CPython
|
||||
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
||||
Classifier: Topic :: Internet :: WWW/HTTP
|
||||
Classifier: Topic :: Software Development :: Libraries
|
||||
Requires-Python: >=3.10
|
||||
Provides-Extra: brotli
|
||||
Requires-Dist: brotli>=1.2.0; (platform_python_implementation == 'CPython') and extra == 'brotli'
|
||||
Requires-Dist: brotlicffi>=1.2.0.0; (platform_python_implementation != 'CPython') and extra == 'brotli'
|
||||
Provides-Extra: h2
|
||||
Requires-Dist: h2<5,>=4; extra == 'h2'
|
||||
Provides-Extra: socks
|
||||
Requires-Dist: pysocks!=1.5.7,<2.0,>=1.5.6; extra == 'socks'
|
||||
Provides-Extra: zstd
|
||||
Requires-Dist: backports-zstd>=1.0.0; (python_version < '3.14') and extra == 'zstd'
|
||||
Description-Content-Type: text/markdown
|
||||
|
||||
<h1 align="center">
|
||||
|
||||

|
||||
|
||||
</h1>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://pypi.org/project/urllib3"><img alt="PyPI Version" src="https://img.shields.io/pypi/v/urllib3.svg?maxAge=86400" /></a>
|
||||
<a href="https://pypi.org/project/urllib3"><img alt="Python Versions" src="https://img.shields.io/pypi/pyversions/urllib3.svg?maxAge=86400" /></a>
|
||||
<a href="https://discord.gg/urllib3"><img alt="Join our Discord" src="https://img.shields.io/discord/756342717725933608?color=%237289da&label=discord" /></a>
|
||||
<a href="https://github.com/urllib3/urllib3/actions?query=workflow%3ACI"><img alt="Coverage Status" src="https://img.shields.io/badge/coverage-100%25-success" /></a>
|
||||
<a href="https://github.com/urllib3/urllib3/actions/workflows/ci.yml?query=branch%3Amain"><img alt="Build Status on GitHub" src="https://github.com/urllib3/urllib3/actions/workflows/ci.yml/badge.svg?branch:main&workflow:CI" /></a>
|
||||
<a href="https://urllib3.readthedocs.io"><img alt="Documentation Status" src="https://readthedocs.org/projects/urllib3/badge/?version=latest" /></a><br>
|
||||
<a href="https://deps.dev/pypi/urllib3"><img alt="OpenSSF Scorecard" src="https://api.securityscorecards.dev/projects/github.com/urllib3/urllib3/badge" /></a>
|
||||
<a href="https://slsa.dev"><img alt="SLSA 3" src="https://slsa.dev/images/gh-badge-level3.svg" /></a>
|
||||
<a href="https://bestpractices.coreinfrastructure.org/projects/6227"><img alt="CII Best Practices" src="https://bestpractices.coreinfrastructure.org/projects/6227/badge" /></a>
|
||||
</p>
|
||||
|
||||
urllib3 is a powerful, *user-friendly* HTTP client for Python.
|
||||
urllib3 brings many critical features that are missing from the Python
|
||||
standard libraries:
|
||||
|
||||
- Thread safety.
|
||||
- Connection pooling.
|
||||
- Client-side SSL/TLS verification.
|
||||
- File uploads with multipart encoding.
|
||||
- Helpers for retrying requests and dealing with HTTP redirects.
|
||||
- Support for gzip, deflate, brotli, and zstd encoding.
|
||||
- Proxy support for HTTP and SOCKS.
|
||||
- 100% test coverage.
|
||||
|
||||
... and many more features, but most importantly: Our maintainers have a 15+
|
||||
year track record of maintaining urllib3 with the highest code standards and
|
||||
attention to security and safety.
|
||||
|
||||
[Much of the Python ecosystem already uses urllib3](https://urllib3.readthedocs.io/en/stable/#who-uses)
|
||||
and you should too.
|
||||
|
||||
|
||||
## Installing
|
||||
|
||||
urllib3 can be installed with [pip](https://pip.pypa.io):
|
||||
|
||||
```bash
|
||||
$ python -m pip install urllib3
|
||||
```
|
||||
|
||||
Alternatively, you can grab the latest source code from [GitHub](https://github.com/urllib3/urllib3):
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/urllib3/urllib3.git
|
||||
$ cd urllib3
|
||||
$ pip install .
|
||||
```
|
||||
|
||||
## Getting Started
|
||||
|
||||
urllib3 is easy to use:
|
||||
|
||||
```python3
|
||||
>>> import urllib3
|
||||
>>> resp = urllib3.request("GET", "http://httpbin.org/robots.txt")
|
||||
>>> resp.status
|
||||
200
|
||||
>>> resp.data
|
||||
b"User-agent: *\nDisallow: /deny\n"
|
||||
```
|
||||
|
||||
urllib3 has usage and reference documentation at [urllib3.readthedocs.io](https://urllib3.readthedocs.io).
|
||||
|
||||
|
||||
## Community
|
||||
|
||||
urllib3 has a [community Discord channel](https://discord.gg/urllib3) for asking questions and
|
||||
collaborating with other contributors. Drop by and say hello 👋
|
||||
|
||||
|
||||
## Contributing
|
||||
|
||||
urllib3 happily accepts contributions. Please see our
|
||||
[contributing documentation](https://urllib3.readthedocs.io/en/latest/contributing.html)
|
||||
for some tips on getting started.
|
||||
|
||||
|
||||
## Security Disclosures
|
||||
|
||||
To report a security vulnerability, please use the
|
||||
[Tidelift security contact](https://tidelift.com/security).
|
||||
Tidelift will coordinate the fix and disclosure with maintainers.
|
||||
|
||||
|
||||
## Maintainers
|
||||
|
||||
Meet our maintainers since 2008:
|
||||
|
||||
- Current Lead: [@illia-v](https://github.com/illia-v) (Illia Volochii)
|
||||
- [@sethmlarson](https://github.com/sethmlarson) (Seth M. Larson)
|
||||
- [@pquentin](https://github.com/pquentin) (Quentin Pradet)
|
||||
- [@theacodes](https://github.com/theacodes) (Thea Flowers)
|
||||
- [@haikuginger](https://github.com/haikuginger) (Jess Shapiro)
|
||||
- [@lukasa](https://github.com/lukasa) (Cory Benfield)
|
||||
- [@sigmavirus24](https://github.com/sigmavirus24) (Ian Stapleton Cordasco)
|
||||
- [@shazow](https://github.com/shazow) (Andrey Petrov)
|
||||
|
||||
👋
|
||||
|
||||
|
||||
## Sponsorship
|
||||
|
||||
If your company benefits from this library, please consider [sponsoring its
|
||||
development](https://urllib3.readthedocs.io/en/latest/sponsors.html).
|
||||
|
||||
|
||||
## For Enterprise
|
||||
|
||||
Professional support for urllib3 is available as part of the [Tidelift
|
||||
Subscription][1]. Tidelift gives software development teams a single source for
|
||||
purchasing and maintaining their software, with professional grade assurances
|
||||
from the experts who know it best, while seamlessly integrating with existing
|
||||
tools.
|
||||
|
||||
[1]: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme
|
||||
@@ -1,79 +0,0 @@
|
||||
urllib3-2.7.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
|
||||
urllib3-2.7.0.dist-info/METADATA,sha256=ZhR4VtMLu2vtAcbOMybQ9I2E7V8oasF3YzoUhrgurOU,6852
|
||||
urllib3-2.7.0.dist-info/RECORD,,
|
||||
urllib3-2.7.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
||||
urllib3-2.7.0.dist-info/licenses/LICENSE.txt,sha256=Ew46ZNX91dCWp1JpRjSn2d8oRGnehuVzIQAmgEHj1oY,1093
|
||||
urllib3/__init__.py,sha256=JMo1tg1nIV1AeJ2vENC_Txfl0e5h6Gzl9DGVk1rWRbo,6979
|
||||
urllib3/__pycache__/__init__.cpython-312.pyc,,
|
||||
urllib3/__pycache__/_base_connection.cpython-312.pyc,,
|
||||
urllib3/__pycache__/_collections.cpython-312.pyc,,
|
||||
urllib3/__pycache__/_request_methods.cpython-312.pyc,,
|
||||
urllib3/__pycache__/_version.cpython-312.pyc,,
|
||||
urllib3/__pycache__/connection.cpython-312.pyc,,
|
||||
urllib3/__pycache__/connectionpool.cpython-312.pyc,,
|
||||
urllib3/__pycache__/exceptions.cpython-312.pyc,,
|
||||
urllib3/__pycache__/fields.cpython-312.pyc,,
|
||||
urllib3/__pycache__/filepost.cpython-312.pyc,,
|
||||
urllib3/__pycache__/poolmanager.cpython-312.pyc,,
|
||||
urllib3/__pycache__/response.cpython-312.pyc,,
|
||||
urllib3/_base_connection.py,sha256=HzcSEHexgDrRUr60jNniB2KQjdw97SedD5luRfHtCXg,5580
|
||||
urllib3/_collections.py,sha256=aOVm2mKilvuvT1efAGtkA7pi65C1NC3HJfpFxvYBS8A,17522
|
||||
urllib3/_request_methods.py,sha256=gCeF85SO_UU4WoPwYHIoz_tw-eM_EVOkLFp8OFsC7DA,9931
|
||||
urllib3/_version.py,sha256=-gMrKhsPiOj0XzUezVFa59d1S6QgQiKgQT5gGgyM8-w,520
|
||||
urllib3/connection.py,sha256=Zos3qxKDW9-GQ6aVqfBQVRM5soB0IjHxY_xXWpJaFTI,42786
|
||||
urllib3/connectionpool.py,sha256=sGFnddXYwlx7KC4JCP1gKvdNGLNK-YTCJDdGACGj3Y8,44164
|
||||
urllib3/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
||||
urllib3/contrib/__pycache__/__init__.cpython-312.pyc,,
|
||||
urllib3/contrib/__pycache__/pyopenssl.cpython-312.pyc,,
|
||||
urllib3/contrib/__pycache__/socks.cpython-312.pyc,,
|
||||
urllib3/contrib/emscripten/__init__.py,sha256=wyXve8rmqX7s2KqRQBxD5Wl48jzWPn5-1u_XoQBELVc,836
|
||||
urllib3/contrib/emscripten/__pycache__/__init__.cpython-312.pyc,,
|
||||
urllib3/contrib/emscripten/__pycache__/connection.cpython-312.pyc,,
|
||||
urllib3/contrib/emscripten/__pycache__/fetch.cpython-312.pyc,,
|
||||
urllib3/contrib/emscripten/__pycache__/request.cpython-312.pyc,,
|
||||
urllib3/contrib/emscripten/__pycache__/response.cpython-312.pyc,,
|
||||
urllib3/contrib/emscripten/connection.py,sha256=giElsBoUsKVURbZzb8GCrJmqW23Xnvj2aNyQVF42slg,8960
|
||||
urllib3/contrib/emscripten/emscripten_fetch_worker.js,sha256=z1k3zZ4_hDKd3-tN7wzz8LHjHC2pxN_uu8B3k9D9A3c,3677
|
||||
urllib3/contrib/emscripten/fetch.py,sha256=5xcd--viFxZd2nBy0aK73dtJ9Tsh1yYZU_SUXwnwibk,23520
|
||||
urllib3/contrib/emscripten/request.py,sha256=mL28szy1KvE3NJhWor5jNmarp8gwplDU-7gwGZY5g0Q,566
|
||||
urllib3/contrib/emscripten/response.py,sha256=CDpY0GFoluR3IxECkM2gH5uDfYDM0EL1FWr7B76wtgM,9719
|
||||
urllib3/contrib/pyopenssl.py,sha256=XZY-QzyT7s8d43qisA4o-eSZYBeIkPyBMZhtj2kkLZs,19734
|
||||
urllib3/contrib/socks.py,sha256=rSuklfha4-vto-8qSLhSPmf5lmRPIYuqdAxKe9bg2RA,7639
|
||||
urllib3/exceptions.py,sha256=hQPHoqo4yw46esNSVkciVQXVUgAxWAglsh6MbyQta-Q,9945
|
||||
urllib3/fields.py,sha256=aGLFAVZpVU-FbJlllve4Ahg0-pH9ZZBQ2Iz7lfpkjkM,10801
|
||||
urllib3/filepost.py,sha256=U8eNZ-mpKKHhrlbHEEiTxxgK16IejhEa7uz42yqA_dI,2388
|
||||
urllib3/http2/__init__.py,sha256=xzrASH7R5ANRkPJOot5lGnATOq3KKuyXzI42rcnwmqs,1741
|
||||
urllib3/http2/__pycache__/__init__.cpython-312.pyc,,
|
||||
urllib3/http2/__pycache__/connection.cpython-312.pyc,,
|
||||
urllib3/http2/__pycache__/probe.cpython-312.pyc,,
|
||||
urllib3/http2/connection.py,sha256=bHMH6fNvatwXPrKqrcn74yA3pUWcqPDppnK1LcKCbP8,12578
|
||||
urllib3/http2/probe.py,sha256=nnAkqbhAakOiF75rz7W0udZ38Eeh_uD8fjV74N73FEI,3014
|
||||
urllib3/poolmanager.py,sha256=c0rh0rcUC1t5tDGuUeGIgAzlErasPOwWwLiB67lX8pM,23895
|
||||
urllib3/py.typed,sha256=UaCuPFa3H8UAakbt-5G8SPacldTOGvJv18pPjUJ5gDY,93
|
||||
urllib3/response.py,sha256=9SX4BkkdoLgsx1ne6hpt-5PncrnDzPzeqC1DaShSc-M,53219
|
||||
urllib3/util/__init__.py,sha256=-qeS0QceivazvBEKDNFCAI-6ACcdDOE4TMvo7SLNlAQ,1001
|
||||
urllib3/util/__pycache__/__init__.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/connection.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/proxy.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/request.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/response.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/retry.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/ssl_.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/ssl_match_hostname.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/ssltransport.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/timeout.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/url.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/util.cpython-312.pyc,,
|
||||
urllib3/util/__pycache__/wait.cpython-312.pyc,,
|
||||
urllib3/util/connection.py,sha256=JjO722lzHlzLXPTkr9ZWBdhseXnMVjMSb1DJLVrXSnQ,4444
|
||||
urllib3/util/proxy.py,sha256=seP8-Q5B6bB0dMtwPj-YcZZQ30vHuLqRu-tI0JZ2fzs,1148
|
||||
urllib3/util/request.py,sha256=itpnC8ug7D4nVfDmGUCRMlgkARUQ13r_XMxSnzTwmpE,8363
|
||||
urllib3/util/response.py,sha256=vQE639uoEhj1vpjEdxu5lNIhJCSUZkd7pqllUI0BZOA,3374
|
||||
urllib3/util/retry.py,sha256=2YnSX-_FecMShD61Mx5s68J0_btUHZrrc_BkFVRS1P4,19577
|
||||
urllib3/util/ssl_.py,sha256=Oqe3rIhUU3e3GVgZob2hxmR8Q0ZDhrhESPlPP_GLaVQ,17742
|
||||
urllib3/util/ssl_match_hostname.py,sha256=Ft44KJzTzGMmKff_ZXP91li2V4WhmvEy16PKBcv4vZk,5479
|
||||
urllib3/util/ssltransport.py,sha256=Ez4O8pR_vT8dan_FvqBYS6dgDfBXEMfVfrzcdUoWfi4,8847
|
||||
urllib3/util/timeout.py,sha256=4eT1FVeZZU7h7mYD1Jq2OXNe4fxekdNvhoWUkZusRpA,10346
|
||||
urllib3/util/url.py,sha256=WRh-TMYXosmgp8m8lT4H5spoHw5yUjlcMCfU53AkoAs,15205
|
||||
urllib3/util/util.py,sha256=j3lbZK1jPyiwD34T8IgJzdWEZVT-4E-0vYIJi9UjeNA,1146
|
||||
urllib3/util/wait.py,sha256=_ph8IrUR3sqPqi0OopQgJUlH4wzkGeM5CiyA7XGGtmI,4423
|
||||
@@ -1,4 +0,0 @@
|
||||
Wheel-Version: 1.0
|
||||
Generator: hatchling 1.29.0
|
||||
Root-Is-Purelib: true
|
||||
Tag: py3-none-any
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2008-2020 Andrey Petrov and contributors.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,49 +1,40 @@
|
||||
"""
|
||||
Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
# Set default logging handler to avoid "No handler found" warnings.
|
||||
import logging
|
||||
import sys
|
||||
import typing
|
||||
import warnings
|
||||
from logging import NullHandler
|
||||
|
||||
from . import exceptions
|
||||
from ._base_connection import _TYPE_BODY
|
||||
from ._collections import HTTPHeaderDict
|
||||
from ._version import __version__
|
||||
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
|
||||
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
|
||||
from .filepost import encode_multipart_formdata
|
||||
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
|
||||
from .response import BaseHTTPResponse, HTTPResponse
|
||||
from .response import HTTPResponse
|
||||
from .util.request import make_headers
|
||||
from .util.retry import Retry
|
||||
from .util.timeout import Timeout
|
||||
from .util.url import get_host
|
||||
|
||||
# Ensure that Python is compiled with OpenSSL 1.1.1+
|
||||
# If the 'ssl' module isn't available at all that's
|
||||
# fine, we only care if the module is available.
|
||||
# === NOTE TO REPACKAGERS AND VENDORS ===
|
||||
# Please delete this block, this logic is only
|
||||
# for urllib3 being distributed via PyPI.
|
||||
# See: https://github.com/urllib3/urllib3/issues/2680
|
||||
try:
|
||||
import ssl
|
||||
import urllib3_secure_extra # type: ignore # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive:
|
||||
warnings.warn(
|
||||
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
|
||||
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
|
||||
"See: https://github.com/urllib3/urllib3/issues/3020",
|
||||
exceptions.NotOpenSSLWarning,
|
||||
)
|
||||
elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive:
|
||||
raise ImportError(
|
||||
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
|
||||
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
|
||||
"See: https://github.com/urllib3/urllib3/issues/2168"
|
||||
)
|
||||
warnings.warn(
|
||||
"'urllib3[secure]' extra is deprecated and will be removed "
|
||||
"in a future release of urllib3 2.x. Read more in this issue: "
|
||||
"https://github.com/urllib3/urllib3/issues/2680",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
__author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
|
||||
__license__ = "MIT"
|
||||
@@ -51,7 +42,6 @@ __version__ = __version__
|
||||
|
||||
__all__ = (
|
||||
"HTTPConnectionPool",
|
||||
"HTTPHeaderDict",
|
||||
"HTTPSConnectionPool",
|
||||
"PoolManager",
|
||||
"ProxyManager",
|
||||
@@ -62,18 +52,15 @@ __all__ = (
|
||||
"connection_from_url",
|
||||
"disable_warnings",
|
||||
"encode_multipart_formdata",
|
||||
"get_host",
|
||||
"make_headers",
|
||||
"proxy_from_url",
|
||||
"request",
|
||||
"BaseHTTPResponse",
|
||||
)
|
||||
|
||||
logging.getLogger(__name__).addHandler(NullHandler())
|
||||
|
||||
|
||||
def add_stderr_logger(
|
||||
level: int = logging.DEBUG,
|
||||
) -> logging.StreamHandler[typing.TextIO]:
|
||||
def add_stderr_logger(level=logging.DEBUG):
|
||||
"""
|
||||
Helper for quickly adding a StreamHandler to the logger. Useful for
|
||||
debugging.
|
||||
@@ -100,112 +87,16 @@ del NullHandler
|
||||
# mechanisms to silence them.
|
||||
# SecurityWarning's always go off by default.
|
||||
warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
|
||||
# SubjectAltNameWarning's should go off once per host
|
||||
warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True)
|
||||
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
|
||||
warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
|
||||
# SNIMissingWarnings should go off only once.
|
||||
warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True)
|
||||
|
||||
|
||||
def disable_warnings(category: type[Warning] = exceptions.HTTPWarning) -> None:
|
||||
def disable_warnings(category=exceptions.HTTPWarning):
|
||||
"""
|
||||
Helper for quickly disabling all urllib3 warnings.
|
||||
"""
|
||||
warnings.simplefilter("ignore", category)
|
||||
|
||||
|
||||
_DEFAULT_POOL = PoolManager()
|
||||
|
||||
|
||||
def request(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
body: _TYPE_BODY | None = None,
|
||||
fields: _TYPE_FIELDS | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
preload_content: bool | None = True,
|
||||
decode_content: bool | None = True,
|
||||
redirect: bool | None = True,
|
||||
retries: Retry | bool | int | None = None,
|
||||
timeout: Timeout | float | int | None = 3,
|
||||
json: typing.Any | None = None,
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
A convenience, top-level request method. It uses a module-global ``PoolManager`` instance.
|
||||
Therefore, its side effects could be shared across dependencies relying on it.
|
||||
To avoid side effects create a new ``PoolManager`` instance and use it instead.
|
||||
The method does not accept low-level ``**urlopen_kw`` keyword arguments.
|
||||
|
||||
:param method:
|
||||
HTTP request method (such as GET, POST, PUT, etc.)
|
||||
|
||||
:param url:
|
||||
The URL to perform the request on.
|
||||
|
||||
:param body:
|
||||
Data to send in the request body, either :class:`str`, :class:`bytes`,
|
||||
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
|
||||
|
||||
:param fields:
|
||||
Data to encode and send in the request body.
|
||||
|
||||
:param headers:
|
||||
Dictionary of custom headers to send, such as User-Agent,
|
||||
If-None-Match, etc.
|
||||
|
||||
:param bool preload_content:
|
||||
If True, the response's body will be preloaded into memory.
|
||||
|
||||
:param bool decode_content:
|
||||
If True, will attempt to decode the body based on the
|
||||
'content-encoding' header.
|
||||
|
||||
:param redirect:
|
||||
If True, automatically handle redirects (status codes 301, 302,
|
||||
303, 307, 308). Each redirect counts as a retry. Disabling retries
|
||||
will disable redirect, too.
|
||||
|
||||
:param retries:
|
||||
Configure the number of retries to allow before raising a
|
||||
:class:`~urllib3.exceptions.MaxRetryError` exception.
|
||||
|
||||
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
|
||||
:class:`~urllib3.util.retry.Retry` object for fine-grained control
|
||||
over different types of retries.
|
||||
Pass an integer number to retry connection errors that many times,
|
||||
but no other types of errors. Pass zero to never retry.
|
||||
|
||||
If ``False``, then retries are disabled and any exception is raised
|
||||
immediately. Also, instead of raising a MaxRetryError on redirects,
|
||||
the redirect response will be returned.
|
||||
|
||||
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
|
||||
|
||||
:param timeout:
|
||||
If specified, overrides the default timeout for this one
|
||||
request. It may be a float (in seconds) or an instance of
|
||||
:class:`urllib3.util.Timeout`.
|
||||
|
||||
:param json:
|
||||
Data to encode and send as JSON with UTF-encoded in the request body.
|
||||
The ``"Content-Type"`` header will be set to ``"application/json"``
|
||||
unless specified otherwise.
|
||||
"""
|
||||
|
||||
return _DEFAULT_POOL.request(
|
||||
method,
|
||||
url,
|
||||
body=body,
|
||||
fields=fields,
|
||||
headers=headers,
|
||||
preload_content=preload_content,
|
||||
decode_content=decode_content,
|
||||
redirect=redirect,
|
||||
retries=retries,
|
||||
timeout=timeout,
|
||||
json=json,
|
||||
)
|
||||
|
||||
|
||||
if sys.platform == "emscripten":
|
||||
from .contrib.emscripten import inject_into_urllib3 # noqa: 401
|
||||
|
||||
inject_into_urllib3()
|
||||
|
||||
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,167 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from .util.connection import _TYPE_SOCKET_OPTIONS
|
||||
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
|
||||
from .util.url import Url
|
||||
|
||||
_TYPE_BODY = typing.Union[
|
||||
bytes, typing.IO[typing.Any], typing.Iterable[bytes | str], str
|
||||
]
|
||||
|
||||
|
||||
class ProxyConfig(typing.NamedTuple):
|
||||
ssl_context: ssl.SSLContext | None
|
||||
use_forwarding_for_https: bool
|
||||
assert_hostname: None | str | typing.Literal[False]
|
||||
assert_fingerprint: str | None
|
||||
|
||||
|
||||
class _ResponseOptions(typing.NamedTuple):
|
||||
# TODO: Remove this in favor of a better
|
||||
# HTTP request/response lifecycle tracking.
|
||||
request_method: str
|
||||
request_url: str
|
||||
preload_content: bool
|
||||
decode_content: bool
|
||||
enforce_content_length: bool
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
import ssl
|
||||
from typing import Protocol
|
||||
|
||||
from .response import BaseHTTPResponse
|
||||
|
||||
class BaseHTTPConnection(Protocol):
|
||||
default_port: typing.ClassVar[int]
|
||||
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
|
||||
|
||||
host: str
|
||||
port: int
|
||||
timeout: None | (
|
||||
float
|
||||
) # Instance doesn't store _DEFAULT_TIMEOUT, must be resolved.
|
||||
blocksize: int
|
||||
source_address: tuple[str, int] | None
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None
|
||||
|
||||
proxy: Url | None
|
||||
proxy_config: ProxyConfig | None
|
||||
|
||||
is_verified: bool
|
||||
proxy_is_verified: bool | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
*,
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
blocksize: int = 8192,
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
|
||||
proxy: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
) -> None: ...
|
||||
|
||||
def set_tunnel(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
scheme: str = "http",
|
||||
) -> None: ...
|
||||
|
||||
def connect(self) -> None: ...
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
# We know *at least* botocore is depending on the order of the
|
||||
# first 3 parameters so to be safe we only mark the later ones
|
||||
# as keyword-only to ensure we have space to extend.
|
||||
*,
|
||||
chunked: bool = False,
|
||||
preload_content: bool = True,
|
||||
decode_content: bool = True,
|
||||
enforce_content_length: bool = True,
|
||||
) -> None: ...
|
||||
|
||||
def getresponse(self) -> BaseHTTPResponse: ...
|
||||
|
||||
def close(self) -> None: ...
|
||||
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
"""Whether the connection either is brand new or has been previously closed.
|
||||
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
|
||||
properties must be False.
|
||||
"""
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Whether the connection is actively connected to any origin (proxy or target)"""
|
||||
|
||||
@property
|
||||
def has_connected_to_proxy(self) -> bool:
|
||||
"""Whether the connection has successfully connected to its proxy.
|
||||
This returns False if no proxy is in use. Used to determine whether
|
||||
errors are coming from the proxy layer or from tunnelling to the target origin.
|
||||
"""
|
||||
|
||||
class BaseHTTPSConnection(BaseHTTPConnection, Protocol):
|
||||
default_port: typing.ClassVar[int]
|
||||
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
|
||||
|
||||
# Certificate verification methods
|
||||
cert_reqs: int | str | None
|
||||
assert_hostname: None | str | typing.Literal[False]
|
||||
assert_fingerprint: str | None
|
||||
ssl_context: ssl.SSLContext | None
|
||||
|
||||
# Trusted CAs
|
||||
ca_certs: str | None
|
||||
ca_cert_dir: str | None
|
||||
ca_cert_data: None | str | bytes
|
||||
|
||||
# TLS version
|
||||
ssl_minimum_version: int | None
|
||||
ssl_maximum_version: int | None
|
||||
ssl_version: int | str | None # Deprecated
|
||||
|
||||
# Client certificates
|
||||
cert_file: str | None
|
||||
key_file: str | None
|
||||
key_password: str | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
*,
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
blocksize: int = 16384,
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
|
||||
proxy: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
cert_reqs: int | str | None = None,
|
||||
assert_hostname: None | str | typing.Literal[False] = None,
|
||||
assert_fingerprint: str | None = None,
|
||||
server_hostname: str | None = None,
|
||||
ssl_context: ssl.SSLContext | None = None,
|
||||
ca_certs: str | None = None,
|
||||
ca_cert_dir: str | None = None,
|
||||
ca_cert_data: None | str | bytes = None,
|
||||
ssl_minimum_version: int | None = None,
|
||||
ssl_maximum_version: int | None = None,
|
||||
ssl_version: int | str | None = None, # Deprecated
|
||||
cert_file: str | None = None,
|
||||
key_file: str | None = None,
|
||||
key_password: str | None = None,
|
||||
) -> None: ...
|
||||
@@ -1,66 +1,34 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
try:
|
||||
from collections.abc import Mapping, MutableMapping
|
||||
except ImportError:
|
||||
from collections import Mapping, MutableMapping
|
||||
try:
|
||||
from threading import RLock
|
||||
except ImportError: # Platform-specific: No threads available
|
||||
|
||||
class RLock:
|
||||
def __enter__(self):
|
||||
pass
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
pass
|
||||
|
||||
|
||||
import typing
|
||||
from collections import OrderedDict
|
||||
from enum import Enum, auto
|
||||
from threading import RLock
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
# We can only import Protocol if TYPE_CHECKING because it's a development
|
||||
# dependency, and is not available at runtime.
|
||||
from typing import Protocol
|
||||
|
||||
from typing_extensions import Self
|
||||
|
||||
class HasGettableStringKeys(Protocol):
|
||||
def keys(self) -> typing.Iterator[str]: ...
|
||||
|
||||
def __getitem__(self, key: str) -> str: ...
|
||||
|
||||
from .exceptions import InvalidHeader
|
||||
from .packages import six
|
||||
from .packages.six import iterkeys, itervalues
|
||||
|
||||
__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
|
||||
|
||||
|
||||
# Key type
|
||||
_KT = typing.TypeVar("_KT")
|
||||
# Value type
|
||||
_VT = typing.TypeVar("_VT")
|
||||
# Default type
|
||||
_DT = typing.TypeVar("_DT")
|
||||
|
||||
ValidHTTPHeaderSource = typing.Union[
|
||||
"HTTPHeaderDict",
|
||||
typing.Mapping[str, str],
|
||||
typing.Iterable[tuple[str, str]],
|
||||
"HasGettableStringKeys",
|
||||
]
|
||||
_Null = object()
|
||||
|
||||
|
||||
class _Sentinel(Enum):
|
||||
not_passed = auto()
|
||||
|
||||
|
||||
def ensure_can_construct_http_header_dict(
|
||||
potential: object,
|
||||
) -> ValidHTTPHeaderSource | None:
|
||||
if isinstance(potential, HTTPHeaderDict):
|
||||
return potential
|
||||
elif isinstance(potential, typing.Mapping):
|
||||
# Full runtime checking of the contents of a Mapping is expensive, so for the
|
||||
# purposes of typechecking, we assume that any Mapping is the right shape.
|
||||
return typing.cast(typing.Mapping[str, str], potential)
|
||||
elif isinstance(potential, typing.Iterable):
|
||||
# Similarly to Mapping, full runtime checking of the contents of an Iterable is
|
||||
# expensive, so for the purposes of typechecking, we assume that any Iterable
|
||||
# is the right shape.
|
||||
return typing.cast(typing.Iterable[tuple[str, str]], potential)
|
||||
elif hasattr(potential, "keys") and hasattr(potential, "__getitem__"):
|
||||
return typing.cast("HasGettableStringKeys", potential)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT, _VT]):
|
||||
class RecentlyUsedContainer(MutableMapping):
|
||||
"""
|
||||
Provides a thread-safe dict-like container which maintains up to
|
||||
``maxsize`` keys while throwing away the least-recently-used keys beyond
|
||||
@@ -74,134 +42,69 @@ class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT,
|
||||
``dispose_func(value)`` is called. Callback which will get called
|
||||
"""
|
||||
|
||||
_container: typing.OrderedDict[_KT, _VT]
|
||||
_maxsize: int
|
||||
dispose_func: typing.Callable[[_VT], None] | None
|
||||
lock: RLock
|
||||
ContainerCls = OrderedDict
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
maxsize: int = 10,
|
||||
dispose_func: typing.Callable[[_VT], None] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
def __init__(self, maxsize=10, dispose_func=None):
|
||||
self._maxsize = maxsize
|
||||
self.dispose_func = dispose_func
|
||||
self._container = OrderedDict()
|
||||
|
||||
self._container = self.ContainerCls()
|
||||
self.lock = RLock()
|
||||
|
||||
def __getitem__(self, key: _KT) -> _VT:
|
||||
def __getitem__(self, key):
|
||||
# Re-insert the item, moving it to the end of the eviction line.
|
||||
with self.lock:
|
||||
item = self._container.pop(key)
|
||||
self._container[key] = item
|
||||
return item
|
||||
|
||||
def __setitem__(self, key: _KT, value: _VT) -> None:
|
||||
evicted_item = None
|
||||
def __setitem__(self, key, value):
|
||||
evicted_value = _Null
|
||||
with self.lock:
|
||||
# Possibly evict the existing value of 'key'
|
||||
try:
|
||||
# If the key exists, we'll overwrite it, which won't change the
|
||||
# size of the pool. Because accessing a key should move it to
|
||||
# the end of the eviction line, we pop it out first.
|
||||
evicted_item = key, self._container.pop(key)
|
||||
self._container[key] = value
|
||||
except KeyError:
|
||||
# When the key does not exist, we insert the value first so that
|
||||
# evicting works in all cases, including when self._maxsize is 0
|
||||
self._container[key] = value
|
||||
if len(self._container) > self._maxsize:
|
||||
# If we didn't evict an existing value, and we've hit our maximum
|
||||
# size, then we have to evict the least recently used item from
|
||||
# the beginning of the container.
|
||||
evicted_item = self._container.popitem(last=False)
|
||||
evicted_value = self._container.get(key, _Null)
|
||||
self._container[key] = value
|
||||
|
||||
# After releasing the lock on the pool, dispose of any evicted value.
|
||||
if evicted_item is not None and self.dispose_func:
|
||||
_, evicted_value = evicted_item
|
||||
# If we didn't evict an existing value, we might have to evict the
|
||||
# least recently used item from the beginning of the container.
|
||||
if len(self._container) > self._maxsize:
|
||||
_key, evicted_value = self._container.popitem(last=False)
|
||||
|
||||
if self.dispose_func and evicted_value is not _Null:
|
||||
self.dispose_func(evicted_value)
|
||||
|
||||
def __delitem__(self, key: _KT) -> None:
|
||||
def __delitem__(self, key):
|
||||
with self.lock:
|
||||
value = self._container.pop(key)
|
||||
|
||||
if self.dispose_func:
|
||||
self.dispose_func(value)
|
||||
|
||||
def __len__(self) -> int:
|
||||
def __len__(self):
|
||||
with self.lock:
|
||||
return len(self._container)
|
||||
|
||||
def __iter__(self) -> typing.NoReturn:
|
||||
def __iter__(self):
|
||||
raise NotImplementedError(
|
||||
"Iteration over this class is unlikely to be threadsafe."
|
||||
)
|
||||
|
||||
def clear(self) -> None:
|
||||
def clear(self):
|
||||
with self.lock:
|
||||
# Copy pointers to all values, then wipe the mapping
|
||||
values = list(self._container.values())
|
||||
values = list(itervalues(self._container))
|
||||
self._container.clear()
|
||||
|
||||
if self.dispose_func:
|
||||
for value in values:
|
||||
self.dispose_func(value)
|
||||
|
||||
def keys(self) -> set[_KT]: # type: ignore[override]
|
||||
def keys(self):
|
||||
with self.lock:
|
||||
return set(self._container.keys())
|
||||
return list(iterkeys(self._container))
|
||||
|
||||
|
||||
class HTTPHeaderDictItemView(set[tuple[str, str]]):
|
||||
"""
|
||||
HTTPHeaderDict is unusual for a Mapping[str, str] in that it has two modes of
|
||||
address.
|
||||
|
||||
If we directly try to get an item with a particular name, we will get a string
|
||||
back that is the concatenated version of all the values:
|
||||
|
||||
>>> d['X-Header-Name']
|
||||
'Value1, Value2, Value3'
|
||||
|
||||
However, if we iterate over an HTTPHeaderDict's items, we will optionally combine
|
||||
these values based on whether combine=True was called when building up the dictionary
|
||||
|
||||
>>> d = HTTPHeaderDict({"A": "1", "B": "foo"})
|
||||
>>> d.add("A", "2", combine=True)
|
||||
>>> d.add("B", "bar")
|
||||
>>> list(d.items())
|
||||
[
|
||||
('A', '1, 2'),
|
||||
('B', 'foo'),
|
||||
('B', 'bar'),
|
||||
]
|
||||
|
||||
This class conforms to the interface required by the MutableMapping ABC while
|
||||
also giving us the nonstandard iteration behavior we want; items with duplicate
|
||||
keys, ordered by time of first insertion.
|
||||
"""
|
||||
|
||||
_headers: HTTPHeaderDict
|
||||
|
||||
def __init__(self, headers: HTTPHeaderDict) -> None:
|
||||
self._headers = headers
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(list(self._headers.iteritems()))
|
||||
|
||||
def __iter__(self) -> typing.Iterator[tuple[str, str]]:
|
||||
return self._headers.iteritems()
|
||||
|
||||
def __contains__(self, item: object) -> bool:
|
||||
if isinstance(item, tuple) and len(item) == 2:
|
||||
passed_key, passed_val = item
|
||||
if isinstance(passed_key, str) and isinstance(passed_val, str):
|
||||
return self._headers._has_value_for_header(passed_key, passed_val)
|
||||
return False
|
||||
|
||||
|
||||
class HTTPHeaderDict(typing.MutableMapping[str, str]):
|
||||
class HTTPHeaderDict(MutableMapping):
|
||||
"""
|
||||
:param headers:
|
||||
An iterable of field-value pairs. Must not contain multiple field names
|
||||
@@ -235,11 +138,9 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
|
||||
'7'
|
||||
"""
|
||||
|
||||
_container: typing.MutableMapping[str, list[str]]
|
||||
|
||||
def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str):
|
||||
super().__init__()
|
||||
self._container = {} # 'dict' is insert-ordered
|
||||
def __init__(self, headers=None, **kwargs):
|
||||
super(HTTPHeaderDict, self).__init__()
|
||||
self._container = OrderedDict()
|
||||
if headers is not None:
|
||||
if isinstance(headers, HTTPHeaderDict):
|
||||
self._copy_from(headers)
|
||||
@@ -248,155 +149,126 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
|
||||
if kwargs:
|
||||
self.extend(kwargs)
|
||||
|
||||
def __setitem__(self, key: str, val: str) -> None:
|
||||
# avoid a bytes/str comparison by decoding before httplib
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode("latin-1")
|
||||
def __setitem__(self, key, val):
|
||||
self._container[key.lower()] = [key, val]
|
||||
return self._container[key.lower()]
|
||||
|
||||
def __getitem__(self, key: str) -> str:
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode("latin-1")
|
||||
def __getitem__(self, key):
|
||||
val = self._container[key.lower()]
|
||||
return ", ".join(val[1:])
|
||||
|
||||
def __delitem__(self, key: str) -> None:
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode("latin-1")
|
||||
def __delitem__(self, key):
|
||||
del self._container[key.lower()]
|
||||
|
||||
def __contains__(self, key: object) -> bool:
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode("latin-1")
|
||||
if isinstance(key, str):
|
||||
return key.lower() in self._container
|
||||
return False
|
||||
def __contains__(self, key):
|
||||
return key.lower() in self._container
|
||||
|
||||
def setdefault(self, key: str, default: str = "") -> str:
|
||||
return super().setdefault(key, default)
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
||||
if maybe_constructable is None:
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, Mapping) and not hasattr(other, "keys"):
|
||||
return False
|
||||
else:
|
||||
other_as_http_header_dict = type(self)(maybe_constructable)
|
||||
if not isinstance(other, type(self)):
|
||||
other = type(self)(other)
|
||||
return dict((k.lower(), v) for k, v in self.itermerged()) == dict(
|
||||
(k.lower(), v) for k, v in other.itermerged()
|
||||
)
|
||||
|
||||
return {k.lower(): v for k, v in self.itermerged()} == {
|
||||
k.lower(): v for k, v in other_as_http_header_dict.itermerged()
|
||||
}
|
||||
|
||||
def __ne__(self, other: object) -> bool:
|
||||
def __ne__(self, other):
|
||||
return not self.__eq__(other)
|
||||
|
||||
def __len__(self) -> int:
|
||||
if six.PY2: # Python 2
|
||||
iterkeys = MutableMapping.iterkeys
|
||||
itervalues = MutableMapping.itervalues
|
||||
|
||||
__marker = object()
|
||||
|
||||
def __len__(self):
|
||||
return len(self._container)
|
||||
|
||||
def __iter__(self) -> typing.Iterator[str]:
|
||||
def __iter__(self):
|
||||
# Only provide the originally cased names
|
||||
for vals in self._container.values():
|
||||
yield vals[0]
|
||||
|
||||
def discard(self, key: str) -> None:
|
||||
def pop(self, key, default=__marker):
|
||||
"""D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
|
||||
If key is not found, d is returned if given, otherwise KeyError is raised.
|
||||
"""
|
||||
# Using the MutableMapping function directly fails due to the private marker.
|
||||
# Using ordinary dict.pop would expose the internal structures.
|
||||
# So let's reinvent the wheel.
|
||||
try:
|
||||
value = self[key]
|
||||
except KeyError:
|
||||
if default is self.__marker:
|
||||
raise
|
||||
return default
|
||||
else:
|
||||
del self[key]
|
||||
return value
|
||||
|
||||
def discard(self, key):
|
||||
try:
|
||||
del self[key]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def add(self, key: str, val: str, *, combine: bool = False) -> None:
|
||||
def add(self, key, val):
|
||||
"""Adds a (name, value) pair, doesn't overwrite the value if it already
|
||||
exists.
|
||||
|
||||
If this is called with combine=True, instead of adding a new header value
|
||||
as a distinct item during iteration, this will instead append the value to
|
||||
any existing header value with a comma. If no existing header value exists
|
||||
for the key, then the value will simply be added, ignoring the combine parameter.
|
||||
|
||||
>>> headers = HTTPHeaderDict(foo='bar')
|
||||
>>> headers.add('Foo', 'baz')
|
||||
>>> headers['foo']
|
||||
'bar, baz'
|
||||
>>> list(headers.items())
|
||||
[('foo', 'bar'), ('foo', 'baz')]
|
||||
>>> headers.add('foo', 'quz', combine=True)
|
||||
>>> list(headers.items())
|
||||
[('foo', 'bar, baz, quz')]
|
||||
"""
|
||||
# avoid a bytes/str comparison by decoding before httplib
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode("latin-1")
|
||||
key_lower = key.lower()
|
||||
new_vals = [key, val]
|
||||
# Keep the common case aka no item present as fast as possible
|
||||
vals = self._container.setdefault(key_lower, new_vals)
|
||||
if new_vals is not vals:
|
||||
# if there are values here, then there is at least the initial
|
||||
# key/value pair
|
||||
assert len(vals) >= 2
|
||||
if combine:
|
||||
vals[-1] = vals[-1] + ", " + val
|
||||
else:
|
||||
vals.append(val)
|
||||
vals.append(val)
|
||||
|
||||
def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None:
|
||||
def extend(self, *args, **kwargs):
|
||||
"""Generic import function for any type of header-like object.
|
||||
Adapted version of MutableMapping.update in order to insert items
|
||||
with self.add instead of self.__setitem__
|
||||
"""
|
||||
if len(args) > 1:
|
||||
raise TypeError(
|
||||
f"extend() takes at most 1 positional arguments ({len(args)} given)"
|
||||
"extend() takes at most 1 positional "
|
||||
"arguments ({0} given)".format(len(args))
|
||||
)
|
||||
other = args[0] if len(args) >= 1 else ()
|
||||
|
||||
if isinstance(other, HTTPHeaderDict):
|
||||
for key, val in other.iteritems():
|
||||
self.add(key, val)
|
||||
elif isinstance(other, typing.Mapping):
|
||||
for key, val in other.items():
|
||||
self.add(key, val)
|
||||
elif isinstance(other, typing.Iterable):
|
||||
for key, value in other:
|
||||
self.add(key, value)
|
||||
elif hasattr(other, "keys") and hasattr(other, "__getitem__"):
|
||||
# THIS IS NOT A TYPESAFE BRANCH
|
||||
# In this branch, the object has a `keys` attr but is not a Mapping or any of
|
||||
# the other types indicated in the method signature. We do some stuff with
|
||||
# it as though it partially implements the Mapping interface, but we're not
|
||||
# doing that stuff safely AT ALL.
|
||||
elif isinstance(other, Mapping):
|
||||
for key in other:
|
||||
self.add(key, other[key])
|
||||
elif hasattr(other, "keys"):
|
||||
for key in other.keys():
|
||||
self.add(key, other[key])
|
||||
else:
|
||||
for key, value in other:
|
||||
self.add(key, value)
|
||||
|
||||
for key, value in kwargs.items():
|
||||
self.add(key, value)
|
||||
|
||||
@typing.overload
|
||||
def getlist(self, key: str) -> list[str]: ...
|
||||
|
||||
@typing.overload
|
||||
def getlist(self, key: str, default: _DT) -> list[str] | _DT: ...
|
||||
|
||||
def getlist(
|
||||
self, key: str, default: _Sentinel | _DT = _Sentinel.not_passed
|
||||
) -> list[str] | _DT:
|
||||
def getlist(self, key, default=__marker):
|
||||
"""Returns a list of all the values for the named field. Returns an
|
||||
empty list if the key doesn't exist."""
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode("latin-1")
|
||||
try:
|
||||
vals = self._container[key.lower()]
|
||||
except KeyError:
|
||||
if default is _Sentinel.not_passed:
|
||||
# _DT is unbound; empty list is instance of List[str]
|
||||
if default is self.__marker:
|
||||
return []
|
||||
# _DT is bound; default is instance of _DT
|
||||
return default
|
||||
else:
|
||||
# _DT may or may not be bound; vals[1:] is instance of List[str], which
|
||||
# meets our external interface requirement of `Union[List[str], _DT]`.
|
||||
return vals[1:]
|
||||
|
||||
def _prepare_for_method_change(self) -> Self:
|
||||
def _prepare_for_method_change(self):
|
||||
"""
|
||||
Remove content-specific header fields before changing the request
|
||||
method to GET or HEAD according to RFC 9110, Section 15.4.
|
||||
@@ -422,65 +294,62 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
|
||||
# Backwards compatibility for http.cookiejar
|
||||
get_all = getlist
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{type(self).__name__}({dict(self.itermerged())})"
|
||||
def __repr__(self):
|
||||
return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
|
||||
|
||||
def _copy_from(self, other: HTTPHeaderDict) -> None:
|
||||
def _copy_from(self, other):
|
||||
for key in other:
|
||||
val = other.getlist(key)
|
||||
self._container[key.lower()] = [key, *val]
|
||||
if isinstance(val, list):
|
||||
# Don't need to convert tuples
|
||||
val = list(val)
|
||||
self._container[key.lower()] = [key] + val
|
||||
|
||||
def copy(self) -> Self:
|
||||
def copy(self):
|
||||
clone = type(self)()
|
||||
clone._copy_from(self)
|
||||
return clone
|
||||
|
||||
def iteritems(self) -> typing.Iterator[tuple[str, str]]:
|
||||
def iteritems(self):
|
||||
"""Iterate over all header lines, including duplicate ones."""
|
||||
for key in self:
|
||||
vals = self._container[key.lower()]
|
||||
for val in vals[1:]:
|
||||
yield vals[0], val
|
||||
|
||||
def itermerged(self) -> typing.Iterator[tuple[str, str]]:
|
||||
def itermerged(self):
|
||||
"""Iterate over all headers, merging duplicate ones together."""
|
||||
for key in self:
|
||||
val = self._container[key.lower()]
|
||||
yield val[0], ", ".join(val[1:])
|
||||
|
||||
def items(self) -> HTTPHeaderDictItemView: # type: ignore[override]
|
||||
return HTTPHeaderDictItemView(self)
|
||||
def items(self):
|
||||
return list(self.iteritems())
|
||||
|
||||
def _has_value_for_header(self, header_name: str, potential_value: str) -> bool:
|
||||
if header_name in self:
|
||||
return potential_value in self._container[header_name.lower()][1:]
|
||||
return False
|
||||
@classmethod
|
||||
def from_httplib(cls, message): # Python 2
|
||||
"""Read headers from a Python 2 httplib message object."""
|
||||
# python2.7 does not expose a proper API for exporting multiheaders
|
||||
# efficiently. This function re-reads raw lines from the message
|
||||
# object and extracts the multiheaders properly.
|
||||
obs_fold_continued_leaders = (" ", "\t")
|
||||
headers = []
|
||||
|
||||
def __ior__(self, other: object) -> HTTPHeaderDict:
|
||||
# Supports extending a header dict in-place using operator |=
|
||||
# combining items with add instead of __setitem__
|
||||
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
||||
if maybe_constructable is None:
|
||||
return NotImplemented
|
||||
self.extend(maybe_constructable)
|
||||
return self
|
||||
for line in message.headers:
|
||||
if line.startswith(obs_fold_continued_leaders):
|
||||
if not headers:
|
||||
# We received a header line that starts with OWS as described
|
||||
# in RFC-7230 S3.2.4. This indicates a multiline header, but
|
||||
# there exists no previous header to which we can attach it.
|
||||
raise InvalidHeader(
|
||||
"Header continuation with no previous header: %s" % line
|
||||
)
|
||||
else:
|
||||
key, value = headers[-1]
|
||||
headers[-1] = (key, value + " " + line.strip())
|
||||
continue
|
||||
|
||||
def __or__(self, other: object) -> Self:
|
||||
# Supports merging header dicts using operator |
|
||||
# combining items with add instead of __setitem__
|
||||
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
||||
if maybe_constructable is None:
|
||||
return NotImplemented
|
||||
result = self.copy()
|
||||
result.extend(maybe_constructable)
|
||||
return result
|
||||
key, value = line.split(":", 1)
|
||||
headers.append((key, value.strip()))
|
||||
|
||||
def __ror__(self, other: object) -> Self:
|
||||
# Supports merging header dicts using operator | when other is on left side
|
||||
# combining items with add instead of __setitem__
|
||||
maybe_constructable = ensure_can_construct_http_header_dict(other)
|
||||
if maybe_constructable is None:
|
||||
return NotImplemented
|
||||
result = type(self)(maybe_constructable)
|
||||
result.extend(self)
|
||||
return result
|
||||
return cls(headers)
|
||||
|
||||
@@ -1,278 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json as _json
|
||||
import typing
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from ._base_connection import _TYPE_BODY
|
||||
from ._collections import HTTPHeaderDict
|
||||
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
|
||||
from .response import BaseHTTPResponse
|
||||
|
||||
__all__ = ["RequestMethods"]
|
||||
|
||||
_TYPE_ENCODE_URL_FIELDS = typing.Union[
|
||||
typing.Sequence[tuple[str, typing.Union[str, bytes]]],
|
||||
typing.Mapping[str, typing.Union[str, bytes]],
|
||||
]
|
||||
|
||||
|
||||
class RequestMethods:
|
||||
"""
|
||||
Convenience mixin for classes who implement a :meth:`urlopen` method, such
|
||||
as :class:`urllib3.HTTPConnectionPool` and
|
||||
:class:`urllib3.PoolManager`.
|
||||
|
||||
Provides behavior for making common types of HTTP request methods and
|
||||
decides which type of request field encoding to use.
|
||||
|
||||
Specifically,
|
||||
|
||||
:meth:`.request_encode_url` is for sending requests whose fields are
|
||||
encoded in the URL (such as GET, HEAD, DELETE).
|
||||
|
||||
:meth:`.request_encode_body` is for sending requests whose fields are
|
||||
encoded in the *body* of the request using multipart or www-form-urlencoded
|
||||
(such as for POST, PUT, PATCH).
|
||||
|
||||
:meth:`.request` is for making any kind of request, it will look up the
|
||||
appropriate encoding format and use one of the above two methods to make
|
||||
the request.
|
||||
|
||||
Initializer parameters:
|
||||
|
||||
:param headers:
|
||||
Headers to include with all requests, unless other headers are given
|
||||
explicitly.
|
||||
"""
|
||||
|
||||
_encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
|
||||
|
||||
def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None:
|
||||
self.headers = headers or {}
|
||||
|
||||
def urlopen(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
encode_multipart: bool = True,
|
||||
multipart_boundary: str | None = None,
|
||||
**kw: typing.Any,
|
||||
) -> BaseHTTPResponse: # Abstract
|
||||
raise NotImplementedError(
|
||||
"Classes extending RequestMethods must implement "
|
||||
"their own ``urlopen`` method."
|
||||
)
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
fields: _TYPE_FIELDS | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
json: typing.Any | None = None,
|
||||
**urlopen_kw: typing.Any,
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
Make a request using :meth:`urlopen` with the appropriate encoding of
|
||||
``fields`` based on the ``method`` used.
|
||||
|
||||
This is a convenience method that requires the least amount of manual
|
||||
effort. It can be used in most situations, while still having the
|
||||
option to drop down to more specific methods when necessary, such as
|
||||
:meth:`request_encode_url`, :meth:`request_encode_body`,
|
||||
or even the lowest level :meth:`urlopen`.
|
||||
|
||||
:param method:
|
||||
HTTP request method (such as GET, POST, PUT, etc.)
|
||||
|
||||
:param url:
|
||||
The URL to perform the request on.
|
||||
|
||||
:param body:
|
||||
Data to send in the request body, either :class:`str`, :class:`bytes`,
|
||||
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
|
||||
|
||||
:param fields:
|
||||
Data to encode and send in the URL or request body, depending on ``method``.
|
||||
|
||||
:param headers:
|
||||
Dictionary of custom headers to send, such as User-Agent,
|
||||
If-None-Match, etc. If None, pool headers are used. If provided,
|
||||
these headers completely replace any pool-specific headers.
|
||||
|
||||
:param json:
|
||||
Data to encode and send as JSON with UTF-encoded in the request body.
|
||||
The ``"Content-Type"`` header will be set to ``"application/json"``
|
||||
unless specified otherwise.
|
||||
"""
|
||||
method = method.upper()
|
||||
|
||||
if json is not None and body is not None:
|
||||
raise TypeError(
|
||||
"request got values for both 'body' and 'json' parameters which are mutually exclusive"
|
||||
)
|
||||
|
||||
if json is not None:
|
||||
if headers is None:
|
||||
headers = self.headers
|
||||
|
||||
if not ("content-type" in map(str.lower, headers.keys())):
|
||||
headers = HTTPHeaderDict(headers)
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
if body is not None:
|
||||
urlopen_kw["body"] = body
|
||||
|
||||
if method in self._encode_url_methods:
|
||||
return self.request_encode_url(
|
||||
method,
|
||||
url,
|
||||
fields=fields, # type: ignore[arg-type]
|
||||
headers=headers,
|
||||
**urlopen_kw,
|
||||
)
|
||||
else:
|
||||
return self.request_encode_body(
|
||||
method, url, fields=fields, headers=headers, **urlopen_kw
|
||||
)
|
||||
|
||||
def request_encode_url(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
fields: _TYPE_ENCODE_URL_FIELDS | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
**urlopen_kw: str,
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
Make a request using :meth:`urlopen` with the ``fields`` encoded in
|
||||
the url. This is useful for request methods like GET, HEAD, DELETE, etc.
|
||||
|
||||
:param method:
|
||||
HTTP request method (such as GET, POST, PUT, etc.)
|
||||
|
||||
:param url:
|
||||
The URL to perform the request on.
|
||||
|
||||
:param fields:
|
||||
Data to encode and send in the URL.
|
||||
|
||||
:param headers:
|
||||
Dictionary of custom headers to send, such as User-Agent,
|
||||
If-None-Match, etc. If None, pool headers are used. If provided,
|
||||
these headers completely replace any pool-specific headers.
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.headers
|
||||
|
||||
extra_kw: dict[str, typing.Any] = {"headers": headers}
|
||||
extra_kw.update(urlopen_kw)
|
||||
|
||||
if fields:
|
||||
url += "?" + urlencode(fields)
|
||||
|
||||
return self.urlopen(method, url, **extra_kw)
|
||||
|
||||
def request_encode_body(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
fields: _TYPE_FIELDS | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
encode_multipart: bool = True,
|
||||
multipart_boundary: str | None = None,
|
||||
**urlopen_kw: str,
|
||||
) -> BaseHTTPResponse:
|
||||
"""
|
||||
Make a request using :meth:`urlopen` with the ``fields`` encoded in
|
||||
the body. This is useful for request methods like POST, PUT, PATCH, etc.
|
||||
|
||||
When ``encode_multipart=True`` (default), then
|
||||
:func:`urllib3.encode_multipart_formdata` is used to encode
|
||||
the payload with the appropriate content type. Otherwise
|
||||
:func:`urllib.parse.urlencode` is used with the
|
||||
'application/x-www-form-urlencoded' content type.
|
||||
|
||||
Multipart encoding must be used when posting files, and it's reasonably
|
||||
safe to use it in other times too. However, it may break request
|
||||
signing, such as with OAuth.
|
||||
|
||||
Supports an optional ``fields`` parameter of key/value strings AND
|
||||
key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
|
||||
the MIME type is optional. For example::
|
||||
|
||||
fields = {
|
||||
'foo': 'bar',
|
||||
'fakefile': ('foofile.txt', 'contents of foofile'),
|
||||
'realfile': ('barfile.txt', open('realfile').read()),
|
||||
'typedfile': ('bazfile.bin', open('bazfile').read(),
|
||||
'image/jpeg'),
|
||||
'nonamefile': 'contents of nonamefile field',
|
||||
}
|
||||
|
||||
When uploading a file, providing a filename (the first parameter of the
|
||||
tuple) is optional but recommended to best mimic behavior of browsers.
|
||||
|
||||
Note that if ``headers`` are supplied, the 'Content-Type' header will
|
||||
be overwritten because it depends on the dynamic random boundary string
|
||||
which is used to compose the body of the request. The random boundary
|
||||
string can be explicitly set with the ``multipart_boundary`` parameter.
|
||||
|
||||
:param method:
|
||||
HTTP request method (such as GET, POST, PUT, etc.)
|
||||
|
||||
:param url:
|
||||
The URL to perform the request on.
|
||||
|
||||
:param fields:
|
||||
Data to encode and send in the request body.
|
||||
|
||||
:param headers:
|
||||
Dictionary of custom headers to send, such as User-Agent,
|
||||
If-None-Match, etc. If None, pool headers are used. If provided,
|
||||
these headers completely replace any pool-specific headers.
|
||||
|
||||
:param encode_multipart:
|
||||
If True, encode the ``fields`` using the multipart/form-data MIME
|
||||
format.
|
||||
|
||||
:param multipart_boundary:
|
||||
If not specified, then a random boundary will be generated using
|
||||
:func:`urllib3.filepost.choose_boundary`.
|
||||
"""
|
||||
if headers is None:
|
||||
headers = self.headers
|
||||
|
||||
extra_kw: dict[str, typing.Any] = {"headers": HTTPHeaderDict(headers)}
|
||||
body: bytes | str
|
||||
|
||||
if fields:
|
||||
if "body" in urlopen_kw:
|
||||
raise TypeError(
|
||||
"request got values for both 'fields' and 'body', can only specify one."
|
||||
)
|
||||
|
||||
if encode_multipart:
|
||||
body, content_type = encode_multipart_formdata(
|
||||
fields, boundary=multipart_boundary
|
||||
)
|
||||
else:
|
||||
body, content_type = (
|
||||
urlencode(fields), # type: ignore[arg-type]
|
||||
"application/x-www-form-urlencoded",
|
||||
)
|
||||
|
||||
extra_kw["body"] = body
|
||||
extra_kw["headers"].setdefault("Content-Type", content_type)
|
||||
|
||||
extra_kw.update(urlopen_kw)
|
||||
|
||||
return self.urlopen(method, url, **extra_kw)
|
||||
@@ -1,24 +1,2 @@
|
||||
# file generated by vcs-versioning
|
||||
# don't change, don't track in version control
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
"__version__",
|
||||
"__version_tuple__",
|
||||
"version",
|
||||
"version_tuple",
|
||||
"__commit_id__",
|
||||
"commit_id",
|
||||
]
|
||||
|
||||
version: str
|
||||
__version__: str
|
||||
__version_tuple__: tuple[int | str, ...]
|
||||
version_tuple: tuple[int | str, ...]
|
||||
commit_id: str | None
|
||||
__commit_id__: str | None
|
||||
|
||||
__version__ = version = '2.7.0'
|
||||
__version_tuple__ = version_tuple = (2, 7, 0)
|
||||
|
||||
__commit_id__ = commit_id = None
|
||||
# This file is protected via CODEOWNERS
|
||||
__version__ = "1.26.20"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -1,17 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import urllib3.connection
|
||||
|
||||
from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||
from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection
|
||||
|
||||
|
||||
def inject_into_urllib3() -> None:
|
||||
# override connection classes to use emscripten specific classes
|
||||
# n.b. mypy complains about the overriding of classes below
|
||||
# if it isn't ignored
|
||||
HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection
|
||||
HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection
|
||||
urllib3.connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment]
|
||||
urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment]
|
||||
urllib3.connection.VerifiedHTTPSConnection = EmscriptenHTTPSConnection # type: ignore[assignment]
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -1,260 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import typing
|
||||
|
||||
# use http.client.HTTPException for consistency with non-emscripten
|
||||
from http.client import HTTPException as HTTPException # noqa: F401
|
||||
from http.client import ResponseNotReady
|
||||
|
||||
from ..._base_connection import _TYPE_BODY
|
||||
from ...connection import HTTPConnection, ProxyConfig, port_by_scheme
|
||||
from ...exceptions import TimeoutError
|
||||
from ...response import BaseHTTPResponse
|
||||
from ...util.connection import _TYPE_SOCKET_OPTIONS
|
||||
from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
|
||||
from ...util.url import Url
|
||||
from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request
|
||||
from .request import EmscriptenRequest
|
||||
from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
|
||||
|
||||
|
||||
class EmscriptenHTTPConnection:
|
||||
default_port: typing.ClassVar[int] = port_by_scheme["http"]
|
||||
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
|
||||
|
||||
timeout: None | (float)
|
||||
|
||||
host: str
|
||||
port: int
|
||||
blocksize: int
|
||||
source_address: tuple[str, int] | None
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None
|
||||
|
||||
proxy: Url | None
|
||||
proxy_config: ProxyConfig | None
|
||||
|
||||
is_verified: bool = False
|
||||
proxy_is_verified: bool | None = None
|
||||
|
||||
response_class: type[BaseHTTPResponse] = EmscriptenHttpResponseWrapper
|
||||
_response: EmscriptenResponse | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int = 0,
|
||||
*,
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
blocksize: int = 8192,
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None = None,
|
||||
proxy: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.timeout = timeout if isinstance(timeout, float) else 0.0
|
||||
self.scheme = "http"
|
||||
self._closed = True
|
||||
self._response = None
|
||||
# ignore these things because we don't
|
||||
# have control over that stuff
|
||||
self.proxy = None
|
||||
self.proxy_config = None
|
||||
self.blocksize = blocksize
|
||||
self.source_address = None
|
||||
self.socket_options = None
|
||||
self.is_verified = False
|
||||
|
||||
def set_tunnel(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = 0,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
scheme: str = "http",
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
def connect(self) -> None:
|
||||
pass
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
# We know *at least* botocore is depending on the order of the
|
||||
# first 3 parameters so to be safe we only mark the later ones
|
||||
# as keyword-only to ensure we have space to extend.
|
||||
*,
|
||||
chunked: bool = False,
|
||||
preload_content: bool = True,
|
||||
decode_content: bool = True,
|
||||
enforce_content_length: bool = True,
|
||||
) -> None:
|
||||
self._closed = False
|
||||
if url.startswith("/"):
|
||||
if self.port is not None:
|
||||
port = f":{self.port}"
|
||||
else:
|
||||
port = ""
|
||||
# no scheme / host / port included, make a full url
|
||||
url = f"{self.scheme}://{self.host}{port}{url}"
|
||||
request = EmscriptenRequest(
|
||||
url=url,
|
||||
method=method,
|
||||
timeout=self.timeout if self.timeout else 0,
|
||||
decode_content=decode_content,
|
||||
)
|
||||
request.set_body(body)
|
||||
if headers:
|
||||
for k, v in headers.items():
|
||||
request.set_header(k, v)
|
||||
self._response = None
|
||||
try:
|
||||
if not preload_content:
|
||||
self._response = send_streaming_request(request)
|
||||
if self._response is None:
|
||||
self._response = send_request(request)
|
||||
except _TimeoutError as e:
|
||||
raise TimeoutError(e.message) from e
|
||||
except _RequestError as e:
|
||||
raise HTTPException(e.message) from e
|
||||
|
||||
def getresponse(self) -> BaseHTTPResponse:
|
||||
if self._response is not None:
|
||||
return EmscriptenHttpResponseWrapper(
|
||||
internal_response=self._response,
|
||||
url=self._response.request.url,
|
||||
connection=self,
|
||||
)
|
||||
else:
|
||||
raise ResponseNotReady()
|
||||
|
||||
def close(self) -> None:
|
||||
self._closed = True
|
||||
self._response = None
|
||||
|
||||
@property
|
||||
def is_closed(self) -> bool:
|
||||
"""Whether the connection either is brand new or has been previously closed.
|
||||
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
|
||||
properties must be False.
|
||||
"""
|
||||
return self._closed
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Whether the connection is actively connected to any origin (proxy or target)"""
|
||||
return True
|
||||
|
||||
@property
|
||||
def has_connected_to_proxy(self) -> bool:
|
||||
"""Whether the connection has successfully connected to its proxy.
|
||||
This returns False if no proxy is in use. Used to determine whether
|
||||
errors are coming from the proxy layer or from tunnelling to the target origin.
|
||||
"""
|
||||
return False
|
||||
|
||||
|
||||
class EmscriptenHTTPSConnection(EmscriptenHTTPConnection):
|
||||
default_port = port_by_scheme["https"]
|
||||
# all this is basically ignored, as browser handles https
|
||||
cert_reqs: int | str | None = None
|
||||
ca_certs: str | None = None
|
||||
ca_cert_dir: str | None = None
|
||||
ca_cert_data: None | str | bytes = None
|
||||
cert_file: str | None
|
||||
key_file: str | None
|
||||
key_password: str | None
|
||||
ssl_context: typing.Any | None
|
||||
ssl_version: int | str | None = None
|
||||
ssl_minimum_version: int | None = None
|
||||
ssl_maximum_version: int | None = None
|
||||
assert_hostname: None | str | typing.Literal[False]
|
||||
assert_fingerprint: str | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int = 0,
|
||||
*,
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
blocksize: int = 16384,
|
||||
socket_options: (
|
||||
None | _TYPE_SOCKET_OPTIONS
|
||||
) = HTTPConnection.default_socket_options,
|
||||
proxy: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
cert_reqs: int | str | None = None,
|
||||
assert_hostname: None | str | typing.Literal[False] = None,
|
||||
assert_fingerprint: str | None = None,
|
||||
server_hostname: str | None = None,
|
||||
ssl_context: typing.Any | None = None,
|
||||
ca_certs: str | None = None,
|
||||
ca_cert_dir: str | None = None,
|
||||
ca_cert_data: None | str | bytes = None,
|
||||
ssl_minimum_version: int | None = None,
|
||||
ssl_maximum_version: int | None = None,
|
||||
ssl_version: int | str | None = None, # Deprecated
|
||||
cert_file: str | None = None,
|
||||
key_file: str | None = None,
|
||||
key_password: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
host,
|
||||
port=port,
|
||||
timeout=timeout,
|
||||
source_address=source_address,
|
||||
blocksize=blocksize,
|
||||
socket_options=socket_options,
|
||||
proxy=proxy,
|
||||
proxy_config=proxy_config,
|
||||
)
|
||||
self.scheme = "https"
|
||||
|
||||
self.key_file = key_file
|
||||
self.cert_file = cert_file
|
||||
self.key_password = key_password
|
||||
self.ssl_context = ssl_context
|
||||
self.server_hostname = server_hostname
|
||||
self.assert_hostname = assert_hostname
|
||||
self.assert_fingerprint = assert_fingerprint
|
||||
self.ssl_version = ssl_version
|
||||
self.ssl_minimum_version = ssl_minimum_version
|
||||
self.ssl_maximum_version = ssl_maximum_version
|
||||
self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
|
||||
self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
|
||||
self.ca_cert_data = ca_cert_data
|
||||
|
||||
self.cert_reqs = None
|
||||
|
||||
# The browser will automatically verify all requests.
|
||||
# We have no control over that setting.
|
||||
self.is_verified = True
|
||||
|
||||
def set_cert(
|
||||
self,
|
||||
key_file: str | None = None,
|
||||
cert_file: str | None = None,
|
||||
cert_reqs: int | str | None = None,
|
||||
key_password: str | None = None,
|
||||
ca_certs: str | None = None,
|
||||
assert_hostname: None | str | typing.Literal[False] = None,
|
||||
assert_fingerprint: str | None = None,
|
||||
ca_cert_dir: str | None = None,
|
||||
ca_cert_data: None | str | bytes = None,
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
# verify that this class implements BaseHTTP(s) connection correctly
|
||||
if typing.TYPE_CHECKING:
|
||||
_supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0)
|
||||
_supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0)
|
||||
-110
@@ -1,110 +0,0 @@
|
||||
let Status = {
|
||||
SUCCESS_HEADER: -1,
|
||||
SUCCESS_EOF: -2,
|
||||
ERROR_TIMEOUT: -3,
|
||||
ERROR_EXCEPTION: -4,
|
||||
};
|
||||
|
||||
let connections = new Map();
|
||||
let nextConnectionID = 1;
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
self.addEventListener("message", async function (event) {
|
||||
if (event.data.close) {
|
||||
let connectionID = event.data.close;
|
||||
connections.delete(connectionID);
|
||||
return;
|
||||
} else if (event.data.getMore) {
|
||||
let connectionID = event.data.getMore;
|
||||
let { curOffset, value, reader, intBuffer, byteBuffer } =
|
||||
connections.get(connectionID);
|
||||
// if we still have some in buffer, then just send it back straight away
|
||||
if (!value || curOffset >= value.length) {
|
||||
// read another buffer if required
|
||||
try {
|
||||
let readResponse = await reader.read();
|
||||
|
||||
if (readResponse.done) {
|
||||
// read everything - clear connection and return
|
||||
connections.delete(connectionID);
|
||||
Atomics.store(intBuffer, 0, Status.SUCCESS_EOF);
|
||||
Atomics.notify(intBuffer, 0);
|
||||
// finished reading successfully
|
||||
// return from event handler
|
||||
return;
|
||||
}
|
||||
curOffset = 0;
|
||||
connections.get(connectionID).value = readResponse.value;
|
||||
value = readResponse.value;
|
||||
} catch (error) {
|
||||
console.log("Request exception:", error);
|
||||
let errorBytes = encoder.encode(error.message);
|
||||
let written = errorBytes.length;
|
||||
byteBuffer.set(errorBytes);
|
||||
intBuffer[1] = written;
|
||||
Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION);
|
||||
Atomics.notify(intBuffer, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// send as much buffer as we can
|
||||
let curLen = value.length - curOffset;
|
||||
if (curLen > byteBuffer.length) {
|
||||
curLen = byteBuffer.length;
|
||||
}
|
||||
byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0);
|
||||
|
||||
Atomics.store(intBuffer, 0, curLen); // store current length in bytes
|
||||
Atomics.notify(intBuffer, 0);
|
||||
curOffset += curLen;
|
||||
connections.get(connectionID).curOffset = curOffset;
|
||||
|
||||
return;
|
||||
} else {
|
||||
// start fetch
|
||||
let connectionID = nextConnectionID;
|
||||
nextConnectionID += 1;
|
||||
const intBuffer = new Int32Array(event.data.buffer);
|
||||
const byteBuffer = new Uint8Array(event.data.buffer, 8);
|
||||
try {
|
||||
const response = await fetch(event.data.url, event.data.fetchParams);
|
||||
// return the headers first via textencoder
|
||||
var headers = [];
|
||||
for (const pair of response.headers.entries()) {
|
||||
headers.push([pair[0], pair[1]]);
|
||||
}
|
||||
let headerObj = {
|
||||
headers: headers,
|
||||
status: response.status,
|
||||
connectionID,
|
||||
};
|
||||
const headerText = JSON.stringify(headerObj);
|
||||
let headerBytes = encoder.encode(headerText);
|
||||
let written = headerBytes.length;
|
||||
byteBuffer.set(headerBytes);
|
||||
intBuffer[1] = written;
|
||||
// make a connection
|
||||
connections.set(connectionID, {
|
||||
reader: response.body.getReader(),
|
||||
intBuffer: intBuffer,
|
||||
byteBuffer: byteBuffer,
|
||||
value: undefined,
|
||||
curOffset: 0,
|
||||
});
|
||||
// set header ready
|
||||
Atomics.store(intBuffer, 0, Status.SUCCESS_HEADER);
|
||||
Atomics.notify(intBuffer, 0);
|
||||
// all fetching after this goes through a new postmessage call with getMore
|
||||
// this allows for parallel requests
|
||||
} catch (error) {
|
||||
console.log("Request exception:", error);
|
||||
let errorBytes = encoder.encode(error.message);
|
||||
let written = errorBytes.length;
|
||||
byteBuffer.set(errorBytes);
|
||||
intBuffer[1] = written;
|
||||
Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION);
|
||||
Atomics.notify(intBuffer, 0);
|
||||
}
|
||||
}
|
||||
});
|
||||
self.postMessage({ inited: true });
|
||||
@@ -1,726 +0,0 @@
|
||||
"""
|
||||
Support for streaming http requests in emscripten.
|
||||
|
||||
A few caveats -
|
||||
|
||||
If your browser (or Node.js) has WebAssembly JavaScript Promise Integration enabled
|
||||
https://github.com/WebAssembly/js-promise-integration/blob/main/proposals/js-promise-integration/Overview.md
|
||||
*and* you launch pyodide using `pyodide.runPythonAsync`, this will fetch data using the
|
||||
JavaScript asynchronous fetch api (wrapped via `pyodide.ffi.call_sync`). In this case
|
||||
timeouts and streaming should just work.
|
||||
|
||||
Otherwise, it uses a combination of XMLHttpRequest and a web-worker for streaming.
|
||||
|
||||
This approach has several caveats:
|
||||
|
||||
Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed.
|
||||
Streaming only works if you're running pyodide in a web worker.
|
||||
|
||||
Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch
|
||||
operation, so it requires that you have crossOriginIsolation enabled, by serving over https
|
||||
(or from localhost) with the two headers below set:
|
||||
|
||||
Cross-Origin-Opener-Policy: same-origin
|
||||
Cross-Origin-Embedder-Policy: require-corp
|
||||
|
||||
You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in
|
||||
JavaScript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole
|
||||
request into a buffer and then returning it. it shows a warning in the JavaScript console in this case.
|
||||
|
||||
Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once
|
||||
control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch.
|
||||
|
||||
NB: in this code, there are a lot of JavaScript objects. They are named js_*
|
||||
to make it clear what type of object they are.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from email.parser import Parser
|
||||
from importlib.resources import files
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import js # type: ignore[import-not-found]
|
||||
from pyodide.ffi import ( # type: ignore[import-not-found]
|
||||
JsArray,
|
||||
JsException,
|
||||
JsProxy,
|
||||
to_js,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing_extensions import Buffer
|
||||
|
||||
from .request import EmscriptenRequest
|
||||
from .response import EmscriptenResponse
|
||||
|
||||
"""
|
||||
There are some headers that trigger unintended CORS preflight requests.
|
||||
See also https://github.com/koenvo/pyodide-http/issues/22
|
||||
"""
|
||||
HEADERS_TO_IGNORE = ("user-agent",)
|
||||
|
||||
SUCCESS_HEADER = -1
|
||||
SUCCESS_EOF = -2
|
||||
ERROR_TIMEOUT = -3
|
||||
ERROR_EXCEPTION = -4
|
||||
|
||||
|
||||
class _RequestError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
message: str | None = None,
|
||||
*,
|
||||
request: EmscriptenRequest | None = None,
|
||||
response: EmscriptenResponse | None = None,
|
||||
):
|
||||
self.request = request
|
||||
self.response = response
|
||||
self.message = message
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
class _StreamingError(_RequestError):
|
||||
pass
|
||||
|
||||
|
||||
class _TimeoutError(_RequestError):
|
||||
pass
|
||||
|
||||
|
||||
def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy:
|
||||
return to_js(dict_val, dict_converter=js.Object.fromEntries)
|
||||
|
||||
|
||||
class _ReadStream(io.RawIOBase):
|
||||
def __init__(
|
||||
self,
|
||||
int_buffer: JsArray,
|
||||
byte_buffer: JsArray,
|
||||
timeout: float,
|
||||
worker: JsProxy,
|
||||
connection_id: int,
|
||||
request: EmscriptenRequest,
|
||||
):
|
||||
self.int_buffer = int_buffer
|
||||
self.byte_buffer = byte_buffer
|
||||
self.read_pos = 0
|
||||
self.read_len = 0
|
||||
self.connection_id = connection_id
|
||||
self.worker = worker
|
||||
self.timeout = int(1000 * timeout) if timeout > 0 else None
|
||||
self.is_live = True
|
||||
self._is_closed = False
|
||||
self.request: EmscriptenRequest | None = request
|
||||
|
||||
def __del__(self) -> None:
|
||||
self.close()
|
||||
|
||||
# this is compatible with _base_connection
|
||||
def is_closed(self) -> bool:
|
||||
return self._is_closed
|
||||
|
||||
# for compatibility with RawIOBase
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return self.is_closed()
|
||||
|
||||
def close(self) -> None:
|
||||
if self.is_closed():
|
||||
return
|
||||
self.read_len = 0
|
||||
self.read_pos = 0
|
||||
self.int_buffer = None
|
||||
self.byte_buffer = None
|
||||
self._is_closed = True
|
||||
self.request = None
|
||||
if self.is_live:
|
||||
self.worker.postMessage(_obj_from_dict({"close": self.connection_id}))
|
||||
self.is_live = False
|
||||
super().close()
|
||||
|
||||
def readable(self) -> bool:
|
||||
return True
|
||||
|
||||
def writable(self) -> bool:
|
||||
return False
|
||||
|
||||
def seekable(self) -> bool:
|
||||
return False
|
||||
|
||||
def readinto(self, byte_obj: Buffer) -> int:
|
||||
if not self.int_buffer:
|
||||
raise _StreamingError(
|
||||
"No buffer for stream in _ReadStream.readinto",
|
||||
request=self.request,
|
||||
response=None,
|
||||
)
|
||||
if self.read_len == 0:
|
||||
# wait for the worker to send something
|
||||
js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT)
|
||||
self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id}))
|
||||
if (
|
||||
js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout)
|
||||
== "timed-out"
|
||||
):
|
||||
raise _TimeoutError
|
||||
data_len = self.int_buffer[0]
|
||||
if data_len > 0:
|
||||
self.read_len = data_len
|
||||
self.read_pos = 0
|
||||
elif data_len == ERROR_EXCEPTION:
|
||||
string_len = self.int_buffer[1]
|
||||
# decode the error string
|
||||
js_decoder = js.TextDecoder.new()
|
||||
json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len))
|
||||
raise _StreamingError(
|
||||
f"Exception thrown in fetch: {json_str}",
|
||||
request=self.request,
|
||||
response=None,
|
||||
)
|
||||
else:
|
||||
# EOF, free the buffers and return zero
|
||||
# and free the request
|
||||
self.is_live = False
|
||||
self.close()
|
||||
return 0
|
||||
# copy from int32array to python bytes
|
||||
ret_length = min(self.read_len, len(memoryview(byte_obj)))
|
||||
subarray = self.byte_buffer.subarray(
|
||||
self.read_pos, self.read_pos + ret_length
|
||||
).to_py()
|
||||
memoryview(byte_obj)[0:ret_length] = subarray
|
||||
self.read_len -= ret_length
|
||||
self.read_pos += ret_length
|
||||
return ret_length
|
||||
|
||||
|
||||
class _StreamingFetcher:
|
||||
def __init__(self) -> None:
|
||||
# make web-worker and data buffer on startup
|
||||
self.streaming_ready = False
|
||||
streaming_worker_code = (
|
||||
files(__package__)
|
||||
.joinpath("emscripten_fetch_worker.js")
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
js_data_blob = js.Blob.new(
|
||||
to_js([streaming_worker_code], create_pyproxies=False),
|
||||
_obj_from_dict({"type": "application/javascript"}),
|
||||
)
|
||||
|
||||
def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None:
|
||||
def onMsg(e: JsProxy) -> None:
|
||||
self.streaming_ready = True
|
||||
js_resolve_fn(e)
|
||||
|
||||
def onErr(e: JsProxy) -> None:
|
||||
js_reject_fn(e) # Defensive: never happens in ci
|
||||
|
||||
self.js_worker.onmessage = onMsg
|
||||
self.js_worker.onerror = onErr
|
||||
|
||||
js_data_url = js.URL.createObjectURL(js_data_blob)
|
||||
self.js_worker = js.globalThis.Worker.new(js_data_url)
|
||||
self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver)
|
||||
|
||||
def send(self, request: EmscriptenRequest) -> EmscriptenResponse:
|
||||
headers = {
|
||||
k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE
|
||||
}
|
||||
|
||||
body = request.body
|
||||
fetch_data = {"headers": headers, "body": to_js(body), "method": request.method}
|
||||
# start the request off in the worker
|
||||
timeout = int(1000 * request.timeout) if request.timeout > 0 else None
|
||||
js_shared_buffer = js.SharedArrayBuffer.new(1048576)
|
||||
js_int_buffer = js.Int32Array.new(js_shared_buffer)
|
||||
js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8)
|
||||
|
||||
js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT)
|
||||
js.Atomics.notify(js_int_buffer, 0)
|
||||
js_absolute_url = js.URL.new(request.url, js.location).href
|
||||
self.js_worker.postMessage(
|
||||
_obj_from_dict(
|
||||
{
|
||||
"buffer": js_shared_buffer,
|
||||
"url": js_absolute_url,
|
||||
"fetchParams": fetch_data,
|
||||
}
|
||||
)
|
||||
)
|
||||
# wait for the worker to send something
|
||||
js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout)
|
||||
if js_int_buffer[0] == ERROR_TIMEOUT:
|
||||
raise _TimeoutError(
|
||||
"Timeout connecting to streaming request",
|
||||
request=request,
|
||||
response=None,
|
||||
)
|
||||
elif js_int_buffer[0] == SUCCESS_HEADER:
|
||||
# got response
|
||||
# header length is in second int of intBuffer
|
||||
string_len = js_int_buffer[1]
|
||||
# decode the rest to a JSON string
|
||||
js_decoder = js.TextDecoder.new()
|
||||
# this does a copy (the slice) because decode can't work on shared array
|
||||
# for some silly reason
|
||||
json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
|
||||
# get it as an object
|
||||
response_obj = json.loads(json_str)
|
||||
return EmscriptenResponse(
|
||||
request=request,
|
||||
status_code=response_obj["status"],
|
||||
headers=response_obj["headers"],
|
||||
body=_ReadStream(
|
||||
js_int_buffer,
|
||||
js_byte_buffer,
|
||||
request.timeout,
|
||||
self.js_worker,
|
||||
response_obj["connectionID"],
|
||||
request,
|
||||
),
|
||||
)
|
||||
elif js_int_buffer[0] == ERROR_EXCEPTION:
|
||||
string_len = js_int_buffer[1]
|
||||
# decode the error string
|
||||
js_decoder = js.TextDecoder.new()
|
||||
json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
|
||||
raise _StreamingError(
|
||||
f"Exception thrown in fetch: {json_str}", request=request, response=None
|
||||
)
|
||||
else:
|
||||
raise _StreamingError(
|
||||
f"Unknown status from worker in fetch: {js_int_buffer[0]}",
|
||||
request=request,
|
||||
response=None,
|
||||
)
|
||||
|
||||
|
||||
class _JSPIReadStream(io.RawIOBase):
|
||||
"""
|
||||
A read stream that uses pyodide.ffi.run_sync to read from a JavaScript fetch
|
||||
response. This requires support for WebAssembly JavaScript Promise Integration
|
||||
in the containing browser, and for pyodide to be launched via runPythonAsync.
|
||||
|
||||
:param js_read_stream:
|
||||
The JavaScript stream reader
|
||||
|
||||
:param timeout:
|
||||
Timeout in seconds
|
||||
|
||||
:param request:
|
||||
The request we're handling
|
||||
|
||||
:param response:
|
||||
The response this stream relates to
|
||||
|
||||
:param js_abort_controller:
|
||||
A JavaScript AbortController object, used for timeouts
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
js_read_stream: Any,
|
||||
timeout: float,
|
||||
request: EmscriptenRequest,
|
||||
response: EmscriptenResponse,
|
||||
js_abort_controller: Any, # JavaScript AbortController for timeouts
|
||||
):
|
||||
self.js_read_stream = js_read_stream
|
||||
self.timeout = timeout
|
||||
self._is_closed = False
|
||||
self._is_done = False
|
||||
self.request: EmscriptenRequest | None = request
|
||||
self.response: EmscriptenResponse | None = response
|
||||
self.current_buffer = None
|
||||
self.current_buffer_pos = 0
|
||||
self.js_abort_controller = js_abort_controller
|
||||
|
||||
def __del__(self) -> None:
|
||||
self.close()
|
||||
|
||||
# this is compatible with _base_connection
|
||||
def is_closed(self) -> bool:
|
||||
return self._is_closed
|
||||
|
||||
# for compatibility with RawIOBase
|
||||
@property
|
||||
def closed(self) -> bool:
|
||||
return self.is_closed()
|
||||
|
||||
def close(self) -> None:
|
||||
if self.is_closed():
|
||||
return
|
||||
self.read_len = 0
|
||||
self.read_pos = 0
|
||||
self.js_read_stream.cancel()
|
||||
self.js_read_stream = None
|
||||
self._is_closed = True
|
||||
self._is_done = True
|
||||
self.request = None
|
||||
self.response = None
|
||||
super().close()
|
||||
|
||||
def readable(self) -> bool:
|
||||
return True
|
||||
|
||||
def writable(self) -> bool:
|
||||
return False
|
||||
|
||||
def seekable(self) -> bool:
|
||||
return False
|
||||
|
||||
def _get_next_buffer(self) -> bool:
|
||||
result_js = _run_sync_with_timeout(
|
||||
self.js_read_stream.read(),
|
||||
self.timeout,
|
||||
self.js_abort_controller,
|
||||
request=self.request,
|
||||
response=self.response,
|
||||
)
|
||||
if result_js.done:
|
||||
self._is_done = True
|
||||
return False
|
||||
else:
|
||||
self.current_buffer = result_js.value.to_py()
|
||||
self.current_buffer_pos = 0
|
||||
return True
|
||||
|
||||
def readinto(self, byte_obj: Buffer) -> int:
|
||||
if self.current_buffer is None:
|
||||
if not self._get_next_buffer() or self.current_buffer is None:
|
||||
self.close()
|
||||
return 0
|
||||
ret_length = min(
|
||||
len(byte_obj), len(self.current_buffer) - self.current_buffer_pos
|
||||
)
|
||||
byte_obj[0:ret_length] = self.current_buffer[
|
||||
self.current_buffer_pos : self.current_buffer_pos + ret_length
|
||||
]
|
||||
self.current_buffer_pos += ret_length
|
||||
if self.current_buffer_pos == len(self.current_buffer):
|
||||
self.current_buffer = None
|
||||
return ret_length
|
||||
|
||||
|
||||
# check if we are in a worker or not
|
||||
def is_in_browser_main_thread() -> bool:
|
||||
return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window
|
||||
|
||||
|
||||
def is_cross_origin_isolated() -> bool:
|
||||
return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated
|
||||
|
||||
|
||||
def is_in_node() -> bool:
|
||||
return (
|
||||
hasattr(js, "process")
|
||||
and hasattr(js.process, "release")
|
||||
and hasattr(js.process.release, "name")
|
||||
and js.process.release.name == "node"
|
||||
)
|
||||
|
||||
|
||||
def is_worker_available() -> bool:
|
||||
return hasattr(js, "Worker") and hasattr(js, "Blob")
|
||||
|
||||
|
||||
_fetcher: _StreamingFetcher | None = None
|
||||
|
||||
if is_worker_available() and (
|
||||
(is_cross_origin_isolated() and not is_in_browser_main_thread())
|
||||
and (not is_in_node())
|
||||
):
|
||||
_fetcher = _StreamingFetcher()
|
||||
else:
|
||||
_fetcher = None
|
||||
|
||||
|
||||
NODE_JSPI_ERROR = (
|
||||
"urllib3 only works in Node.js with pyodide.runPythonAsync"
|
||||
" and requires the flag --experimental-wasm-stack-switching in "
|
||||
" versions of node <24."
|
||||
)
|
||||
|
||||
|
||||
def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None:
|
||||
if has_jspi():
|
||||
return send_jspi_request(request, True)
|
||||
elif is_in_node():
|
||||
raise _RequestError(
|
||||
message=NODE_JSPI_ERROR,
|
||||
request=request,
|
||||
response=None,
|
||||
)
|
||||
|
||||
if _fetcher and streaming_ready():
|
||||
return _fetcher.send(request)
|
||||
else:
|
||||
_show_streaming_warning()
|
||||
return None
|
||||
|
||||
|
||||
_SHOWN_TIMEOUT_WARNING = False
|
||||
|
||||
|
||||
def _show_timeout_warning() -> None:
|
||||
global _SHOWN_TIMEOUT_WARNING
|
||||
if not _SHOWN_TIMEOUT_WARNING:
|
||||
_SHOWN_TIMEOUT_WARNING = True
|
||||
message = "Warning: Timeout is not available on main browser thread"
|
||||
js.console.warn(message)
|
||||
|
||||
|
||||
_SHOWN_STREAMING_WARNING = False
|
||||
|
||||
|
||||
def _show_streaming_warning() -> None:
|
||||
global _SHOWN_STREAMING_WARNING
|
||||
if not _SHOWN_STREAMING_WARNING:
|
||||
_SHOWN_STREAMING_WARNING = True
|
||||
message = "Can't stream HTTP requests because: \n"
|
||||
if not is_cross_origin_isolated():
|
||||
message += " Page is not cross-origin isolated\n"
|
||||
if is_in_browser_main_thread():
|
||||
message += " Python is running in main browser thread\n"
|
||||
if not is_worker_available():
|
||||
message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in
|
||||
if streaming_ready() is False:
|
||||
message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch
|
||||
is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`"""
|
||||
from js import console
|
||||
|
||||
console.warn(message)
|
||||
|
||||
|
||||
def send_request(request: EmscriptenRequest) -> EmscriptenResponse:
|
||||
if has_jspi():
|
||||
return send_jspi_request(request, False)
|
||||
elif is_in_node():
|
||||
raise _RequestError(
|
||||
message=NODE_JSPI_ERROR,
|
||||
request=request,
|
||||
response=None,
|
||||
)
|
||||
try:
|
||||
js_xhr = js.XMLHttpRequest.new()
|
||||
|
||||
if not is_in_browser_main_thread():
|
||||
js_xhr.responseType = "arraybuffer"
|
||||
if request.timeout:
|
||||
js_xhr.timeout = int(request.timeout * 1000)
|
||||
else:
|
||||
js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15")
|
||||
if request.timeout:
|
||||
# timeout isn't available on the main thread - show a warning in console
|
||||
# if it is set
|
||||
_show_timeout_warning()
|
||||
|
||||
js_xhr.open(request.method, request.url, False)
|
||||
for name, value in request.headers.items():
|
||||
if name.lower() not in HEADERS_TO_IGNORE:
|
||||
js_xhr.setRequestHeader(name, value)
|
||||
|
||||
js_xhr.send(to_js(request.body))
|
||||
|
||||
headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders()))
|
||||
|
||||
if not is_in_browser_main_thread():
|
||||
body = js_xhr.response.to_py().tobytes()
|
||||
else:
|
||||
body = js_xhr.response.encode("ISO-8859-15")
|
||||
return EmscriptenResponse(
|
||||
status_code=js_xhr.status, headers=headers, body=body, request=request
|
||||
)
|
||||
except JsException as err:
|
||||
if err.name == "TimeoutError":
|
||||
raise _TimeoutError(err.message, request=request)
|
||||
elif err.name == "NetworkError":
|
||||
raise _RequestError(err.message, request=request)
|
||||
else:
|
||||
# general http error
|
||||
raise _RequestError(err.message, request=request)
|
||||
|
||||
|
||||
def send_jspi_request(
|
||||
request: EmscriptenRequest, streaming: bool
|
||||
) -> EmscriptenResponse:
|
||||
"""
|
||||
Send a request using WebAssembly JavaScript Promise Integration
|
||||
to wrap the asynchronous JavaScript fetch api (experimental).
|
||||
|
||||
:param request:
|
||||
Request to send
|
||||
|
||||
:param streaming:
|
||||
Whether to stream the response
|
||||
|
||||
:return: The response object
|
||||
:rtype: EmscriptenResponse
|
||||
"""
|
||||
timeout = request.timeout
|
||||
js_abort_controller = js.AbortController.new()
|
||||
headers = {k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE}
|
||||
req_body = request.body
|
||||
fetch_data = {
|
||||
"headers": headers,
|
||||
"body": to_js(req_body),
|
||||
"method": request.method,
|
||||
"signal": js_abort_controller.signal,
|
||||
}
|
||||
# Node.js returns the whole response (unlike opaqueredirect in browsers),
|
||||
# so urllib3 can set `redirect: manual` to control redirects itself.
|
||||
# https://stackoverflow.com/a/78524615
|
||||
if _is_node_js():
|
||||
fetch_data["redirect"] = "manual"
|
||||
# Call JavaScript fetch (async api, returns a promise)
|
||||
fetcher_promise_js = js.fetch(request.url, _obj_from_dict(fetch_data))
|
||||
# Now suspend WebAssembly until we resolve that promise
|
||||
# or time out.
|
||||
response_js = _run_sync_with_timeout(
|
||||
fetcher_promise_js,
|
||||
timeout,
|
||||
js_abort_controller,
|
||||
request=request,
|
||||
response=None,
|
||||
)
|
||||
headers = {}
|
||||
header_iter = response_js.headers.entries()
|
||||
while True:
|
||||
iter_value_js = header_iter.next()
|
||||
if getattr(iter_value_js, "done", False):
|
||||
break
|
||||
else:
|
||||
headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1])
|
||||
status_code = response_js.status
|
||||
body: bytes | io.RawIOBase = b""
|
||||
|
||||
response = EmscriptenResponse(
|
||||
status_code=status_code, headers=headers, body=b"", request=request
|
||||
)
|
||||
if streaming:
|
||||
# get via inputstream
|
||||
if response_js.body is not None:
|
||||
# get a reader from the fetch response
|
||||
body_stream_js = response_js.body.getReader()
|
||||
body = _JSPIReadStream(
|
||||
body_stream_js, timeout, request, response, js_abort_controller
|
||||
)
|
||||
else:
|
||||
# get directly via arraybuffer
|
||||
# n.b. this is another async JavaScript call.
|
||||
body = _run_sync_with_timeout(
|
||||
response_js.arrayBuffer(),
|
||||
timeout,
|
||||
js_abort_controller,
|
||||
request=request,
|
||||
response=response,
|
||||
).to_py()
|
||||
response.body = body
|
||||
return response
|
||||
|
||||
|
||||
def _run_sync_with_timeout(
|
||||
promise: Any,
|
||||
timeout: float,
|
||||
js_abort_controller: Any,
|
||||
request: EmscriptenRequest | None,
|
||||
response: EmscriptenResponse | None,
|
||||
) -> Any:
|
||||
"""
|
||||
Await a JavaScript promise synchronously with a timeout which is implemented
|
||||
via the AbortController
|
||||
|
||||
:param promise:
|
||||
Javascript promise to await
|
||||
|
||||
:param timeout:
|
||||
Timeout in seconds
|
||||
|
||||
:param js_abort_controller:
|
||||
A JavaScript AbortController object, used on timeout
|
||||
|
||||
:param request:
|
||||
The request being handled
|
||||
|
||||
:param response:
|
||||
The response being handled (if it exists yet)
|
||||
|
||||
:raises _TimeoutError: If the request times out
|
||||
:raises _RequestError: If the request raises a JavaScript exception
|
||||
|
||||
:return: The result of awaiting the promise.
|
||||
"""
|
||||
timer_id = None
|
||||
if timeout > 0:
|
||||
timer_id = js.setTimeout(
|
||||
js_abort_controller.abort.bind(js_abort_controller), int(timeout * 1000)
|
||||
)
|
||||
try:
|
||||
from pyodide.ffi import run_sync
|
||||
|
||||
# run_sync here uses WebAssembly JavaScript Promise Integration to
|
||||
# suspend python until the JavaScript promise resolves.
|
||||
return run_sync(promise)
|
||||
except JsException as err:
|
||||
if err.name == "AbortError":
|
||||
raise _TimeoutError(
|
||||
message="Request timed out", request=request, response=response
|
||||
)
|
||||
else:
|
||||
raise _RequestError(message=err.message, request=request, response=response)
|
||||
finally:
|
||||
if timer_id is not None:
|
||||
js.clearTimeout(timer_id)
|
||||
|
||||
|
||||
def has_jspi() -> bool:
|
||||
"""
|
||||
Return true if jspi can be used.
|
||||
|
||||
This requires both browser support and also WebAssembly
|
||||
to be in the correct state - i.e. that the javascript
|
||||
call into python was async not sync.
|
||||
|
||||
:return: True if jspi can be used.
|
||||
:rtype: bool
|
||||
"""
|
||||
try:
|
||||
from pyodide.ffi import can_run_sync, run_sync # noqa: F401
|
||||
|
||||
return bool(can_run_sync())
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
def _is_node_js() -> bool:
|
||||
"""
|
||||
Check if we are in Node.js.
|
||||
|
||||
:return: True if we are in Node.js.
|
||||
:rtype: bool
|
||||
"""
|
||||
return (
|
||||
hasattr(js, "process")
|
||||
and hasattr(js.process, "release")
|
||||
# According to the Node.js documentation, the release name is always "node".
|
||||
and js.process.release.name == "node"
|
||||
)
|
||||
|
||||
|
||||
def streaming_ready() -> bool | None:
|
||||
if _fetcher:
|
||||
return _fetcher.streaming_ready
|
||||
else:
|
||||
return None # no fetcher, return None to signify that
|
||||
|
||||
|
||||
async def wait_for_streaming_ready() -> bool:
|
||||
if _fetcher:
|
||||
await _fetcher.js_worker_ready_promise
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
@@ -1,22 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from ..._base_connection import _TYPE_BODY
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmscriptenRequest:
|
||||
method: str
|
||||
url: str
|
||||
params: dict[str, str] | None = None
|
||||
body: _TYPE_BODY | None = None
|
||||
headers: dict[str, str] = field(default_factory=dict)
|
||||
timeout: float = 0
|
||||
decode_content: bool = True
|
||||
|
||||
def set_header(self, name: str, value: str) -> None:
|
||||
self.headers[name.capitalize()] = value
|
||||
|
||||
def set_body(self, body: _TYPE_BODY | None) -> None:
|
||||
self.body = body
|
||||
@@ -1,281 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json as _json
|
||||
import logging
|
||||
import typing
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from http.client import HTTPException as HTTPException
|
||||
from io import BytesIO, IOBase
|
||||
|
||||
from ...exceptions import InvalidHeader, TimeoutError
|
||||
from ...response import BaseHTTPResponse
|
||||
from ...util.retry import Retry
|
||||
from .request import EmscriptenRequest
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmscriptenResponse:
|
||||
status_code: int
|
||||
headers: dict[str, str]
|
||||
body: IOBase | bytes
|
||||
request: EmscriptenRequest
|
||||
|
||||
|
||||
class EmscriptenHttpResponseWrapper(BaseHTTPResponse):
|
||||
def __init__(
|
||||
self,
|
||||
internal_response: EmscriptenResponse,
|
||||
url: str | None = None,
|
||||
connection: BaseHTTPConnection | BaseHTTPSConnection | None = None,
|
||||
):
|
||||
self._pool = None # set by pool class
|
||||
self._body = None
|
||||
self._uncached_read_occurred = False
|
||||
self._response = internal_response
|
||||
self._url = url
|
||||
self._connection = connection
|
||||
self._closed = False
|
||||
super().__init__(
|
||||
headers=internal_response.headers,
|
||||
status=internal_response.status_code,
|
||||
request_url=url,
|
||||
version=0,
|
||||
version_string="HTTP/?",
|
||||
reason="",
|
||||
decode_content=True,
|
||||
)
|
||||
self.length_remaining = self._init_length(self._response.request.method)
|
||||
self.length_is_certain = False
|
||||
|
||||
@property
|
||||
def url(self) -> str | None:
|
||||
return self._url
|
||||
|
||||
@url.setter
|
||||
def url(self, url: str | None) -> None:
|
||||
self._url = url
|
||||
|
||||
@property
|
||||
def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None:
|
||||
return self._connection
|
||||
|
||||
@property
|
||||
def retries(self) -> Retry | None:
|
||||
return self._retries
|
||||
|
||||
@retries.setter
|
||||
def retries(self, retries: Retry | None) -> None:
|
||||
# Override the request_url if retries has a redirect location.
|
||||
self._retries = retries
|
||||
|
||||
def stream(
|
||||
self, amt: int | None = 2**16, decode_content: bool | None = None
|
||||
) -> typing.Generator[bytes]:
|
||||
"""
|
||||
A generator wrapper for the read() method. A call will block until
|
||||
``amt`` bytes have been read from the connection or until the
|
||||
connection is closed.
|
||||
|
||||
:param amt:
|
||||
How much of the content to read. The generator will return up to
|
||||
much data per iteration, but may return less. This is particularly
|
||||
likely when using compressed data. However, the empty string will
|
||||
never be returned.
|
||||
|
||||
:param decode_content:
|
||||
If True, will attempt to decode the body based on the
|
||||
'content-encoding' header.
|
||||
"""
|
||||
while True:
|
||||
data = self.read(amt=amt, decode_content=decode_content)
|
||||
|
||||
if data:
|
||||
yield data
|
||||
else:
|
||||
break
|
||||
|
||||
def _init_length(self, request_method: str | None) -> int | None:
|
||||
length: int | None
|
||||
content_length: str | None = self.headers.get("content-length")
|
||||
|
||||
if content_length is not None:
|
||||
try:
|
||||
# RFC 7230 section 3.3.2 specifies multiple content lengths can
|
||||
# be sent in a single Content-Length header
|
||||
# (e.g. Content-Length: 42, 42). This line ensures the values
|
||||
# are all valid ints and that as long as the `set` length is 1,
|
||||
# all values are the same. Otherwise, the header is invalid.
|
||||
lengths = {int(val) for val in content_length.split(",")}
|
||||
if len(lengths) > 1:
|
||||
raise InvalidHeader(
|
||||
"Content-Length contained multiple "
|
||||
"unmatching values (%s)" % content_length
|
||||
)
|
||||
length = lengths.pop()
|
||||
except ValueError:
|
||||
length = None
|
||||
else:
|
||||
if length < 0:
|
||||
length = None
|
||||
|
||||
else: # if content_length is None
|
||||
length = None
|
||||
|
||||
# Check for responses that shouldn't include a body
|
||||
if (
|
||||
self.status in (204, 304)
|
||||
or 100 <= self.status < 200
|
||||
or request_method == "HEAD"
|
||||
):
|
||||
length = 0
|
||||
|
||||
return length
|
||||
|
||||
def read(
|
||||
self,
|
||||
amt: int | None = None,
|
||||
decode_content: bool | None = None, # ignored because browser decodes always
|
||||
cache_content: bool = False,
|
||||
) -> bytes:
|
||||
if (
|
||||
self._closed
|
||||
or self._response is None
|
||||
or (isinstance(self._response.body, IOBase) and self._response.body.closed)
|
||||
):
|
||||
return b""
|
||||
|
||||
with self._error_catcher():
|
||||
# body has been preloaded as a string by XmlHttpRequest
|
||||
if not isinstance(self._response.body, IOBase):
|
||||
self.length_remaining = len(self._response.body)
|
||||
self.length_is_certain = True
|
||||
# wrap body in IOStream
|
||||
self._response.body = BytesIO(self._response.body)
|
||||
if amt is not None and amt >= 0:
|
||||
# don't cache partial content
|
||||
cache_content = False
|
||||
data = self._response.body.read(amt)
|
||||
self._uncached_read_occurred = True
|
||||
else: # read all we can (and cache it)
|
||||
data = self._response.body.read()
|
||||
if cache_content and not self._uncached_read_occurred:
|
||||
self._body = data
|
||||
else:
|
||||
self._uncached_read_occurred = True
|
||||
if self.length_remaining is not None:
|
||||
self.length_remaining = max(self.length_remaining - len(data), 0)
|
||||
if len(data) == 0 or (
|
||||
self.length_is_certain and self.length_remaining == 0
|
||||
):
|
||||
# definitely finished reading, close response stream
|
||||
self._response.body.close()
|
||||
return typing.cast(bytes, data)
|
||||
|
||||
def read_chunked(
|
||||
self,
|
||||
amt: int | None = None,
|
||||
decode_content: bool | None = None,
|
||||
) -> typing.Generator[bytes]:
|
||||
# chunked is handled by browser
|
||||
while True:
|
||||
bytes = self.read(amt, decode_content)
|
||||
if not bytes:
|
||||
break
|
||||
yield bytes
|
||||
|
||||
def release_conn(self) -> None:
|
||||
if not self._pool or not self._connection:
|
||||
return None
|
||||
|
||||
self._pool._put_conn(self._connection)
|
||||
self._connection = None
|
||||
|
||||
def drain_conn(self) -> None:
|
||||
self.close()
|
||||
|
||||
@property
|
||||
def data(self) -> bytes:
|
||||
if self._body:
|
||||
return self._body
|
||||
else:
|
||||
return self.read(cache_content=True)
|
||||
|
||||
def json(self) -> typing.Any:
|
||||
"""
|
||||
Deserializes the body of the HTTP response as a Python object.
|
||||
|
||||
The body of the HTTP response must be encoded using UTF-8, as per
|
||||
`RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
|
||||
|
||||
To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
|
||||
your custom decoder instead.
|
||||
|
||||
If the body of the HTTP response is not decodable to UTF-8, a
|
||||
`UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
|
||||
valid JSON document, a `json.JSONDecodeError` will be raised.
|
||||
|
||||
Read more :ref:`here <json_content>`.
|
||||
|
||||
:returns: The body of the HTTP response as a Python object.
|
||||
"""
|
||||
data = self.data.decode("utf-8")
|
||||
return _json.loads(data)
|
||||
|
||||
def close(self) -> None:
|
||||
if not self._closed:
|
||||
if isinstance(self._response.body, IOBase):
|
||||
self._response.body.close()
|
||||
if self._connection:
|
||||
self._connection.close()
|
||||
self._connection = None
|
||||
self._closed = True
|
||||
|
||||
@contextmanager
|
||||
def _error_catcher(self) -> typing.Generator[None]:
|
||||
"""
|
||||
Catch Emscripten specific exceptions thrown by fetch.py,
|
||||
instead re-raising urllib3 variants, so that low-level exceptions
|
||||
are not leaked in the high-level api.
|
||||
|
||||
On exit, release the connection back to the pool.
|
||||
"""
|
||||
from .fetch import _RequestError, _TimeoutError # avoid circular import
|
||||
|
||||
clean_exit = False
|
||||
|
||||
try:
|
||||
yield
|
||||
# If no exception is thrown, we should avoid cleaning up
|
||||
# unnecessarily.
|
||||
clean_exit = True
|
||||
except _TimeoutError as e:
|
||||
raise TimeoutError(str(e))
|
||||
except _RequestError as e:
|
||||
raise HTTPException(str(e))
|
||||
finally:
|
||||
# If we didn't terminate cleanly, we need to throw away our
|
||||
# connection.
|
||||
if not clean_exit:
|
||||
# The response may not be closed but we're not going to use it
|
||||
# anymore so close it now
|
||||
if (
|
||||
isinstance(self._response.body, IOBase)
|
||||
and not self._response.body.closed
|
||||
):
|
||||
self._response.body.close()
|
||||
# release the connection back to the pool
|
||||
self.release_conn()
|
||||
else:
|
||||
# If we have read everything from the response stream,
|
||||
# return the connection back to the pool.
|
||||
if (
|
||||
isinstance(self._response.body, IOBase)
|
||||
and self._response.body.closed
|
||||
):
|
||||
self.release_conn()
|
||||
@@ -1,17 +1,17 @@
|
||||
"""
|
||||
Module for using pyOpenSSL as a TLS backend. This module was relevant before
|
||||
the standard library ``ssl`` module supported SNI, but now that we've dropped
|
||||
support for Python 2.7 all relevant Python versions support SNI so
|
||||
**this module is no longer recommended**.
|
||||
TLS with SNI_-support for Python 2. Follow these instructions if you would
|
||||
like to verify TLS certificates in Python 2. Note, the default libraries do
|
||||
*not* do certificate checking; you need to do additional work to validate
|
||||
certificates yourself.
|
||||
|
||||
This needs the following packages installed:
|
||||
|
||||
* `pyOpenSSL`_ (tested with 19.0.0)
|
||||
* `cryptography`_ (minimum 2.3, from pyopenssl)
|
||||
* `idna`_ (minimum 2.1, from cryptography)
|
||||
* `pyOpenSSL`_ (tested with 16.0.0)
|
||||
* `cryptography`_ (minimum 1.3.4, from pyopenssl)
|
||||
* `idna`_ (minimum 2.0, from cryptography)
|
||||
|
||||
However, pyOpenSSL depends on cryptography, so while we use all three directly here we
|
||||
end up having relatively few packages required.
|
||||
However, pyopenssl depends on cryptography, which depends on idna, so while we
|
||||
use all three directly here we end up having relatively few packages required.
|
||||
|
||||
You can install them with the following command:
|
||||
|
||||
@@ -33,45 +33,75 @@ like this:
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
Now you can use :mod:`urllib3` as you normally would, and it will support SNI
|
||||
when the required modules are installed.
|
||||
|
||||
Activating this module also has the positive side effect of disabling SSL/TLS
|
||||
compression in Python 2 (see `CRIME attack`_).
|
||||
|
||||
.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
|
||||
.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
|
||||
.. _pyopenssl: https://www.pyopenssl.org
|
||||
.. _cryptography: https://cryptography.io
|
||||
.. _idna: https://github.com/kjd/idna
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import OpenSSL.SSL # type: ignore[import-not-found]
|
||||
import OpenSSL.crypto
|
||||
import OpenSSL.SSL
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.backends.openssl import backend as openssl_backend
|
||||
|
||||
try:
|
||||
from cryptography.x509 import UnsupportedExtension # type: ignore[attr-defined]
|
||||
from cryptography.x509 import UnsupportedExtension
|
||||
except ImportError:
|
||||
# UnsupportedExtension is gone in cryptography >= 2.1.0
|
||||
class UnsupportedExtension(Exception): # type: ignore[no-redef]
|
||||
class UnsupportedExtension(Exception):
|
||||
pass
|
||||
|
||||
|
||||
from io import BytesIO
|
||||
from socket import error as SocketError
|
||||
from socket import timeout
|
||||
|
||||
try: # Platform-specific: Python 2
|
||||
from socket import _fileobject
|
||||
except ImportError: # Platform-specific: Python 3
|
||||
_fileobject = None
|
||||
from ..packages.backports.makefile import backport_makefile
|
||||
|
||||
import logging
|
||||
import ssl
|
||||
import typing
|
||||
from io import BytesIO
|
||||
from socket import socket as socket_cls
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
from .. import util
|
||||
from ..packages import six
|
||||
from ..util.ssl_ import PROTOCOL_TLS_CLIENT
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from OpenSSL.crypto import X509 # type: ignore[import-not-found]
|
||||
|
||||
warnings.warn(
|
||||
"'urllib3.contrib.pyopenssl' module is deprecated and will be removed "
|
||||
"in a future release of urllib3 2.x. Read more in this issue: "
|
||||
"https://github.com/urllib3/urllib3/issues/2680",
|
||||
category=DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
|
||||
|
||||
# SNI always works.
|
||||
HAS_SNI = True
|
||||
|
||||
# Map from urllib3 to PyOpenSSL compatible parameter-values.
|
||||
_openssl_versions: dict[int, int] = {
|
||||
util.ssl_.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
|
||||
util.ssl_.PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
|
||||
_openssl_versions = {
|
||||
util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD,
|
||||
PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD,
|
||||
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
|
||||
}
|
||||
|
||||
if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"):
|
||||
_openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD
|
||||
|
||||
if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
|
||||
_openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
|
||||
|
||||
@@ -85,77 +115,43 @@ _stdlib_to_openssl_verify = {
|
||||
ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
|
||||
+ OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
|
||||
}
|
||||
_openssl_to_stdlib_verify = {v: k for k, v in _stdlib_to_openssl_verify.items()}
|
||||
|
||||
# The SSLvX values are the most likely to be missing in the future
|
||||
# but we check them all just to be sure.
|
||||
_OP_NO_SSLv2_OR_SSLv3: int = getattr(OpenSSL.SSL, "OP_NO_SSLv2", 0) | getattr(
|
||||
OpenSSL.SSL, "OP_NO_SSLv3", 0
|
||||
)
|
||||
_OP_NO_TLSv1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1", 0)
|
||||
_OP_NO_TLSv1_1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_1", 0)
|
||||
_OP_NO_TLSv1_2: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_2", 0)
|
||||
_OP_NO_TLSv1_3: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_3", 0)
|
||||
|
||||
_openssl_to_ssl_minimum_version: dict[int, int] = {
|
||||
ssl.TLSVersion.MINIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
|
||||
ssl.TLSVersion.TLSv1: _OP_NO_SSLv2_OR_SSLv3,
|
||||
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1,
|
||||
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1,
|
||||
ssl.TLSVersion.TLSv1_3: (
|
||||
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
|
||||
),
|
||||
ssl.TLSVersion.MAXIMUM_SUPPORTED: (
|
||||
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
|
||||
),
|
||||
}
|
||||
_openssl_to_ssl_maximum_version: dict[int, int] = {
|
||||
ssl.TLSVersion.MINIMUM_SUPPORTED: (
|
||||
_OP_NO_SSLv2_OR_SSLv3
|
||||
| _OP_NO_TLSv1
|
||||
| _OP_NO_TLSv1_1
|
||||
| _OP_NO_TLSv1_2
|
||||
| _OP_NO_TLSv1_3
|
||||
),
|
||||
ssl.TLSVersion.TLSv1: (
|
||||
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3
|
||||
),
|
||||
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3,
|
||||
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_3,
|
||||
ssl.TLSVersion.TLSv1_3: _OP_NO_SSLv2_OR_SSLv3,
|
||||
ssl.TLSVersion.MAXIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
|
||||
}
|
||||
_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items())
|
||||
|
||||
# OpenSSL will only write 16K at a time
|
||||
SSL_WRITE_BLOCKSIZE = 16384
|
||||
|
||||
orig_util_HAS_SNI = util.HAS_SNI
|
||||
orig_util_SSLContext = util.ssl_.SSLContext
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def inject_into_urllib3() -> None:
|
||||
def inject_into_urllib3():
|
||||
"Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
|
||||
|
||||
_validate_dependencies_met()
|
||||
|
||||
util.SSLContext = PyOpenSSLContext # type: ignore[assignment]
|
||||
util.ssl_.SSLContext = PyOpenSSLContext # type: ignore[assignment]
|
||||
util.SSLContext = PyOpenSSLContext
|
||||
util.ssl_.SSLContext = PyOpenSSLContext
|
||||
util.HAS_SNI = HAS_SNI
|
||||
util.ssl_.HAS_SNI = HAS_SNI
|
||||
util.IS_PYOPENSSL = True
|
||||
util.ssl_.IS_PYOPENSSL = True
|
||||
|
||||
|
||||
def extract_from_urllib3() -> None:
|
||||
def extract_from_urllib3():
|
||||
"Undo monkey-patching by :func:`inject_into_urllib3`."
|
||||
|
||||
util.SSLContext = orig_util_SSLContext
|
||||
util.ssl_.SSLContext = orig_util_SSLContext
|
||||
util.HAS_SNI = orig_util_HAS_SNI
|
||||
util.ssl_.HAS_SNI = orig_util_HAS_SNI
|
||||
util.IS_PYOPENSSL = False
|
||||
util.ssl_.IS_PYOPENSSL = False
|
||||
|
||||
|
||||
def _validate_dependencies_met() -> None:
|
||||
def _validate_dependencies_met():
|
||||
"""
|
||||
Verifies that PyOpenSSL's package-level dependencies have been met.
|
||||
Throws `ImportError` if they are not met.
|
||||
@@ -181,7 +177,7 @@ def _validate_dependencies_met() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _dnsname_to_stdlib(name: str) -> str | None:
|
||||
def _dnsname_to_stdlib(name):
|
||||
"""
|
||||
Converts a dNSName SubjectAlternativeName field to the form used by the
|
||||
standard library on the given Python version.
|
||||
@@ -195,7 +191,7 @@ def _dnsname_to_stdlib(name: str) -> str | None:
|
||||
the name given should be skipped.
|
||||
"""
|
||||
|
||||
def idna_encode(name: str) -> bytes | None:
|
||||
def idna_encode(name):
|
||||
"""
|
||||
Borrowed wholesale from the Python Cryptography Project. It turns out
|
||||
that we can't just safely call `idna.encode`: it can explode for
|
||||
@@ -204,7 +200,7 @@ def _dnsname_to_stdlib(name: str) -> str | None:
|
||||
import idna
|
||||
|
||||
try:
|
||||
for prefix in ["*.", "."]:
|
||||
for prefix in [u"*.", u"."]:
|
||||
if name.startswith(prefix):
|
||||
name = name[len(prefix) :]
|
||||
return prefix.encode("ascii") + idna.encode(name)
|
||||
@@ -216,17 +212,24 @@ def _dnsname_to_stdlib(name: str) -> str | None:
|
||||
if ":" in name:
|
||||
return name
|
||||
|
||||
encoded_name = idna_encode(name)
|
||||
if encoded_name is None:
|
||||
name = idna_encode(name)
|
||||
if name is None:
|
||||
return None
|
||||
return encoded_name.decode("utf-8")
|
||||
elif sys.version_info >= (3, 0):
|
||||
name = name.decode("utf-8")
|
||||
return name
|
||||
|
||||
|
||||
def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]:
|
||||
def get_subj_alt_name(peer_cert):
|
||||
"""
|
||||
Given an PyOpenSSL certificate, provides all the subject alternative names.
|
||||
"""
|
||||
cert = peer_cert.to_cryptography()
|
||||
# Pass the cert to cryptography, which has much better APIs for this.
|
||||
if hasattr(peer_cert, "to_cryptography"):
|
||||
cert = peer_cert.to_cryptography()
|
||||
else:
|
||||
der = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, peer_cert)
|
||||
cert = x509.load_der_x509_certificate(der, openssl_backend)
|
||||
|
||||
# We want to find the SAN extension. Ask Cryptography to locate it (it's
|
||||
# faster than looping in Python)
|
||||
@@ -270,94 +273,93 @@ def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]:
|
||||
return names
|
||||
|
||||
|
||||
class WrappedSocket:
|
||||
"""API-compatibility wrapper for Python OpenSSL's Connection-class."""
|
||||
class WrappedSocket(object):
|
||||
"""API-compatibility wrapper for Python OpenSSL's Connection-class.
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
connection: OpenSSL.SSL.Connection,
|
||||
socket: socket_cls,
|
||||
suppress_ragged_eofs: bool = True,
|
||||
) -> None:
|
||||
Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
|
||||
collector of pypy.
|
||||
"""
|
||||
|
||||
def __init__(self, connection, socket, suppress_ragged_eofs=True):
|
||||
self.connection = connection
|
||||
self.socket = socket
|
||||
self.suppress_ragged_eofs = suppress_ragged_eofs
|
||||
self._io_refs = 0
|
||||
self._makefile_refs = 0
|
||||
self._closed = False
|
||||
|
||||
def fileno(self) -> int:
|
||||
def fileno(self):
|
||||
return self.socket.fileno()
|
||||
|
||||
# Copy-pasted from Python 3.5 source code
|
||||
def _decref_socketios(self) -> None:
|
||||
if self._io_refs > 0:
|
||||
self._io_refs -= 1
|
||||
def _decref_socketios(self):
|
||||
if self._makefile_refs > 0:
|
||||
self._makefile_refs -= 1
|
||||
if self._closed:
|
||||
self.close()
|
||||
|
||||
def recv(self, *args: typing.Any, **kwargs: typing.Any) -> bytes:
|
||||
def recv(self, *args, **kwargs):
|
||||
try:
|
||||
data = self.connection.recv(*args, **kwargs)
|
||||
except OpenSSL.SSL.SysCallError as e:
|
||||
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
|
||||
return b""
|
||||
else:
|
||||
raise OSError(e.args[0], str(e)) from e
|
||||
raise SocketError(str(e))
|
||||
except OpenSSL.SSL.ZeroReturnError:
|
||||
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
|
||||
return b""
|
||||
else:
|
||||
raise
|
||||
except OpenSSL.SSL.WantReadError as e:
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
|
||||
raise TimeoutError("The read operation timed out") from e
|
||||
raise timeout("The read operation timed out")
|
||||
else:
|
||||
return self.recv(*args, **kwargs)
|
||||
|
||||
# TLS 1.3 post-handshake authentication
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"read error: {e!r}") from e
|
||||
raise ssl.SSLError("read error: %r" % e)
|
||||
else:
|
||||
return data # type: ignore[no-any-return]
|
||||
return data
|
||||
|
||||
def recv_into(self, *args: typing.Any, **kwargs: typing.Any) -> int:
|
||||
def recv_into(self, *args, **kwargs):
|
||||
try:
|
||||
return self.connection.recv_into(*args, **kwargs) # type: ignore[no-any-return]
|
||||
return self.connection.recv_into(*args, **kwargs)
|
||||
except OpenSSL.SSL.SysCallError as e:
|
||||
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
|
||||
return 0
|
||||
else:
|
||||
raise OSError(e.args[0], str(e)) from e
|
||||
raise SocketError(str(e))
|
||||
except OpenSSL.SSL.ZeroReturnError:
|
||||
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
|
||||
return 0
|
||||
else:
|
||||
raise
|
||||
except OpenSSL.SSL.WantReadError as e:
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
|
||||
raise TimeoutError("The read operation timed out") from e
|
||||
raise timeout("The read operation timed out")
|
||||
else:
|
||||
return self.recv_into(*args, **kwargs)
|
||||
|
||||
# TLS 1.3 post-handshake authentication
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"read error: {e!r}") from e
|
||||
raise ssl.SSLError("read error: %r" % e)
|
||||
|
||||
def settimeout(self, timeout: float) -> None:
|
||||
def settimeout(self, timeout):
|
||||
return self.socket.settimeout(timeout)
|
||||
|
||||
def _send_until_done(self, data: bytes) -> int:
|
||||
def _send_until_done(self, data):
|
||||
while True:
|
||||
try:
|
||||
return self.connection.send(data) # type: ignore[no-any-return]
|
||||
except OpenSSL.SSL.WantWriteError as e:
|
||||
return self.connection.send(data)
|
||||
except OpenSSL.SSL.WantWriteError:
|
||||
if not util.wait_for_write(self.socket, self.socket.gettimeout()):
|
||||
raise TimeoutError() from e
|
||||
raise timeout()
|
||||
continue
|
||||
except OpenSSL.SSL.SysCallError as e:
|
||||
raise OSError(e.args[0], str(e)) from e
|
||||
raise SocketError(str(e))
|
||||
|
||||
def sendall(self, data: bytes) -> None:
|
||||
def sendall(self, data):
|
||||
total_sent = 0
|
||||
while total_sent < len(data):
|
||||
sent = self._send_until_done(
|
||||
@@ -365,151 +367,135 @@ class WrappedSocket:
|
||||
)
|
||||
total_sent += sent
|
||||
|
||||
def shutdown(self, how: int) -> None:
|
||||
try:
|
||||
self.connection.shutdown()
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"shutdown error: {e!r}") from e
|
||||
def shutdown(self):
|
||||
# FIXME rethrow compatible exceptions should we ever use this
|
||||
self.connection.shutdown()
|
||||
|
||||
def close(self) -> None:
|
||||
self._closed = True
|
||||
if self._io_refs <= 0:
|
||||
self._real_close()
|
||||
def close(self):
|
||||
if self._makefile_refs < 1:
|
||||
try:
|
||||
self._closed = True
|
||||
return self.connection.close()
|
||||
except OpenSSL.SSL.Error:
|
||||
return
|
||||
else:
|
||||
self._makefile_refs -= 1
|
||||
|
||||
def _real_close(self) -> None:
|
||||
try:
|
||||
return self.connection.close() # type: ignore[no-any-return]
|
||||
except OpenSSL.SSL.Error:
|
||||
return
|
||||
|
||||
def getpeercert(
|
||||
self, binary_form: bool = False
|
||||
) -> dict[str, list[typing.Any]] | None:
|
||||
def getpeercert(self, binary_form=False):
|
||||
x509 = self.connection.get_peer_certificate()
|
||||
|
||||
if not x509:
|
||||
return x509 # type: ignore[no-any-return]
|
||||
return x509
|
||||
|
||||
if binary_form:
|
||||
return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) # type: ignore[no-any-return]
|
||||
return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509)
|
||||
|
||||
return {
|
||||
"subject": ((("commonName", x509.get_subject().CN),),), # type: ignore[dict-item]
|
||||
"subject": ((("commonName", x509.get_subject().CN),),),
|
||||
"subjectAltName": get_subj_alt_name(x509),
|
||||
}
|
||||
|
||||
def version(self) -> str:
|
||||
return self.connection.get_protocol_version_name() # type: ignore[no-any-return]
|
||||
def version(self):
|
||||
return self.connection.get_protocol_version_name()
|
||||
|
||||
def selected_alpn_protocol(self) -> str | None:
|
||||
alpn_proto = self.connection.get_alpn_proto_negotiated()
|
||||
return alpn_proto.decode() if alpn_proto else None
|
||||
def _reuse(self):
|
||||
self._makefile_refs += 1
|
||||
|
||||
def _drop(self):
|
||||
if self._makefile_refs < 1:
|
||||
self.close()
|
||||
else:
|
||||
self._makefile_refs -= 1
|
||||
|
||||
|
||||
WrappedSocket.makefile = socket_cls.makefile # type: ignore[attr-defined]
|
||||
if _fileobject: # Platform-specific: Python 2
|
||||
|
||||
def makefile(self, mode, bufsize=-1):
|
||||
self._makefile_refs += 1
|
||||
return _fileobject(self, mode, bufsize, close=True)
|
||||
|
||||
else: # Platform-specific: Python 3
|
||||
makefile = backport_makefile
|
||||
|
||||
WrappedSocket.makefile = makefile
|
||||
|
||||
|
||||
class PyOpenSSLContext:
|
||||
class PyOpenSSLContext(object):
|
||||
"""
|
||||
I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
|
||||
for translating the interface of the standard library ``SSLContext`` object
|
||||
to calls into PyOpenSSL.
|
||||
"""
|
||||
|
||||
def __init__(self, protocol: int) -> None:
|
||||
def __init__(self, protocol):
|
||||
self.protocol = _openssl_versions[protocol]
|
||||
self._ctx = OpenSSL.SSL.Context(self.protocol)
|
||||
self._options = 0
|
||||
self.check_hostname = False
|
||||
self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED
|
||||
self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED
|
||||
self._verify_flags: int = ssl.VERIFY_X509_TRUSTED_FIRST
|
||||
|
||||
@property
|
||||
def options(self) -> int:
|
||||
def options(self):
|
||||
return self._options
|
||||
|
||||
@options.setter
|
||||
def options(self, value: int) -> None:
|
||||
def options(self, value):
|
||||
self._options = value
|
||||
self._set_ctx_options()
|
||||
self._ctx.set_options(value)
|
||||
|
||||
@property
|
||||
def verify_flags(self) -> int:
|
||||
return self._verify_flags
|
||||
|
||||
@verify_flags.setter
|
||||
def verify_flags(self, value: int) -> None:
|
||||
self._verify_flags = value
|
||||
self._ctx.get_cert_store().set_flags(self._verify_flags)
|
||||
|
||||
@property
|
||||
def verify_mode(self) -> int:
|
||||
def verify_mode(self):
|
||||
return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
|
||||
|
||||
@verify_mode.setter
|
||||
def verify_mode(self, value: ssl.VerifyMode) -> None:
|
||||
def verify_mode(self, value):
|
||||
self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
|
||||
|
||||
def set_default_verify_paths(self) -> None:
|
||||
def set_default_verify_paths(self):
|
||||
self._ctx.set_default_verify_paths()
|
||||
|
||||
def set_ciphers(self, ciphers: bytes | str) -> None:
|
||||
if isinstance(ciphers, str):
|
||||
def set_ciphers(self, ciphers):
|
||||
if isinstance(ciphers, six.text_type):
|
||||
ciphers = ciphers.encode("utf-8")
|
||||
self._ctx.set_cipher_list(ciphers)
|
||||
|
||||
def load_verify_locations(
|
||||
self,
|
||||
cafile: str | None = None,
|
||||
capath: str | None = None,
|
||||
cadata: bytes | None = None,
|
||||
) -> None:
|
||||
def load_verify_locations(self, cafile=None, capath=None, cadata=None):
|
||||
if cafile is not None:
|
||||
cafile = cafile.encode("utf-8") # type: ignore[assignment]
|
||||
cafile = cafile.encode("utf-8")
|
||||
if capath is not None:
|
||||
capath = capath.encode("utf-8") # type: ignore[assignment]
|
||||
capath = capath.encode("utf-8")
|
||||
try:
|
||||
self._ctx.load_verify_locations(cafile, capath)
|
||||
if cadata is not None:
|
||||
self._ctx.load_verify_locations(BytesIO(cadata))
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"unable to load trusted certificates: {e!r}") from e
|
||||
raise ssl.SSLError("unable to load trusted certificates: %r" % e)
|
||||
|
||||
def load_cert_chain(
|
||||
self,
|
||||
certfile: str,
|
||||
keyfile: str | None = None,
|
||||
password: str | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
self._ctx.use_certificate_chain_file(certfile)
|
||||
if password is not None:
|
||||
if not isinstance(password, bytes):
|
||||
password = password.encode("utf-8") # type: ignore[assignment]
|
||||
self._ctx.set_passwd_cb(lambda *_: password)
|
||||
self._ctx.use_privatekey_file(keyfile or certfile)
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"Unable to load certificate chain: {e!r}") from e
|
||||
def load_cert_chain(self, certfile, keyfile=None, password=None):
|
||||
self._ctx.use_certificate_chain_file(certfile)
|
||||
if password is not None:
|
||||
if not isinstance(password, six.binary_type):
|
||||
password = password.encode("utf-8")
|
||||
self._ctx.set_passwd_cb(lambda *_: password)
|
||||
self._ctx.use_privatekey_file(keyfile or certfile)
|
||||
|
||||
def set_alpn_protocols(self, protocols: list[bytes | str]) -> None:
|
||||
protocols = [util.util.to_bytes(p, "ascii") for p in protocols]
|
||||
return self._ctx.set_alpn_protos(protocols) # type: ignore[no-any-return]
|
||||
def set_alpn_protocols(self, protocols):
|
||||
protocols = [six.ensure_binary(p) for p in protocols]
|
||||
return self._ctx.set_alpn_protos(protocols)
|
||||
|
||||
def wrap_socket(
|
||||
self,
|
||||
sock: socket_cls,
|
||||
server_side: bool = False,
|
||||
do_handshake_on_connect: bool = True,
|
||||
suppress_ragged_eofs: bool = True,
|
||||
server_hostname: bytes | str | None = None,
|
||||
) -> WrappedSocket:
|
||||
sock,
|
||||
server_side=False,
|
||||
do_handshake_on_connect=True,
|
||||
suppress_ragged_eofs=True,
|
||||
server_hostname=None,
|
||||
):
|
||||
cnx = OpenSSL.SSL.Connection(self._ctx, sock)
|
||||
|
||||
# If server_hostname is an IP, don't use it for SNI, per RFC6066 Section 3
|
||||
if server_hostname and not util.ssl_.is_ipaddress(server_hostname):
|
||||
if isinstance(server_hostname, str):
|
||||
server_hostname = server_hostname.encode("utf-8")
|
||||
if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3
|
||||
server_hostname = server_hostname.encode("utf-8")
|
||||
|
||||
if server_hostname is not None:
|
||||
cnx.set_tlsext_host_name(server_hostname)
|
||||
|
||||
cnx.set_connect_state()
|
||||
@@ -517,47 +503,16 @@ class PyOpenSSLContext:
|
||||
while True:
|
||||
try:
|
||||
cnx.do_handshake()
|
||||
except OpenSSL.SSL.WantReadError as e:
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
if not util.wait_for_read(sock, sock.gettimeout()):
|
||||
raise TimeoutError("select timed out") from e
|
||||
raise timeout("select timed out")
|
||||
continue
|
||||
except OpenSSL.SSL.Error as e:
|
||||
raise ssl.SSLError(f"bad handshake: {e!r}") from e
|
||||
raise ssl.SSLError("bad handshake: %r" % e)
|
||||
break
|
||||
|
||||
return WrappedSocket(cnx, sock)
|
||||
|
||||
def _set_ctx_options(self) -> None:
|
||||
self._ctx.set_options(
|
||||
self._options
|
||||
| _openssl_to_ssl_minimum_version[self._minimum_version]
|
||||
| _openssl_to_ssl_maximum_version[self._maximum_version]
|
||||
)
|
||||
|
||||
@property
|
||||
def minimum_version(self) -> int:
|
||||
return self._minimum_version
|
||||
|
||||
@minimum_version.setter
|
||||
def minimum_version(self, minimum_version: int) -> None:
|
||||
self._minimum_version = minimum_version
|
||||
self._set_ctx_options()
|
||||
|
||||
@property
|
||||
def maximum_version(self) -> int:
|
||||
return self._maximum_version
|
||||
|
||||
@maximum_version.setter
|
||||
def maximum_version(self, maximum_version: int) -> None:
|
||||
self._maximum_version = maximum_version
|
||||
self._set_ctx_options()
|
||||
|
||||
|
||||
def _verify_callback(
|
||||
cnx: OpenSSL.SSL.Connection,
|
||||
x509: X509,
|
||||
err_no: int,
|
||||
err_depth: int,
|
||||
return_code: int,
|
||||
) -> bool:
|
||||
def _verify_callback(cnx, x509, err_no, err_depth, return_code):
|
||||
return err_no == 0
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
This module contains provisional support for SOCKS proxies from within
|
||||
urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and
|
||||
@@ -37,11 +38,10 @@ with the proxy:
|
||||
proxy_url="socks5h://<username>:<password>@proxy-host"
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
try:
|
||||
import socks # type: ignore[import-untyped]
|
||||
import socks
|
||||
except ImportError:
|
||||
import warnings
|
||||
|
||||
@@ -51,13 +51,13 @@ except ImportError:
|
||||
(
|
||||
"SOCKS support in urllib3 requires the installation of optional "
|
||||
"dependencies: specifically, PySocks. For more information, see "
|
||||
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies"
|
||||
"https://urllib3.readthedocs.io/en/1.26.x/contrib.html#socks-proxies"
|
||||
),
|
||||
DependencyWarning,
|
||||
)
|
||||
raise
|
||||
|
||||
import typing
|
||||
from socket import error as SocketError
|
||||
from socket import timeout as SocketTimeout
|
||||
|
||||
from ..connection import HTTPConnection, HTTPSConnection
|
||||
@@ -69,16 +69,7 @@ from ..util.url import parse_url
|
||||
try:
|
||||
import ssl
|
||||
except ImportError:
|
||||
ssl = None # type: ignore[assignment]
|
||||
|
||||
|
||||
class _TYPE_SOCKS_OPTIONS(typing.TypedDict):
|
||||
socks_version: int
|
||||
proxy_host: str | None
|
||||
proxy_port: str | None
|
||||
username: str | None
|
||||
password: str | None
|
||||
rdns: bool
|
||||
ssl = None
|
||||
|
||||
|
||||
class SOCKSConnection(HTTPConnection):
|
||||
@@ -86,20 +77,15 @@ class SOCKSConnection(HTTPConnection):
|
||||
A plain-text HTTP connection that connects via a SOCKS proxy.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
_socks_options: _TYPE_SOCKS_OPTIONS,
|
||||
*args: typing.Any,
|
||||
**kwargs: typing.Any,
|
||||
) -> None:
|
||||
self._socks_options = _socks_options
|
||||
super().__init__(*args, **kwargs)
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._socks_options = kwargs.pop("_socks_options")
|
||||
super(SOCKSConnection, self).__init__(*args, **kwargs)
|
||||
|
||||
def _new_conn(self) -> socks.socksocket:
|
||||
def _new_conn(self):
|
||||
"""
|
||||
Establish a new connection via the SOCKS proxy.
|
||||
"""
|
||||
extra_kw: dict[str, typing.Any] = {}
|
||||
extra_kw = {}
|
||||
if self.source_address:
|
||||
extra_kw["source_address"] = self.source_address
|
||||
|
||||
@@ -116,14 +102,15 @@ class SOCKSConnection(HTTPConnection):
|
||||
proxy_password=self._socks_options["password"],
|
||||
proxy_rdns=self._socks_options["rdns"],
|
||||
timeout=self.timeout,
|
||||
**extra_kw,
|
||||
**extra_kw
|
||||
)
|
||||
|
||||
except SocketTimeout as e:
|
||||
except SocketTimeout:
|
||||
raise ConnectTimeoutError(
|
||||
self,
|
||||
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
||||
) from e
|
||||
"Connection to %s timed out. (connect timeout=%s)"
|
||||
% (self.host, self.timeout),
|
||||
)
|
||||
|
||||
except socks.ProxyError as e:
|
||||
# This is fragile as hell, but it seems to be the only way to raise
|
||||
@@ -133,23 +120,22 @@ class SOCKSConnection(HTTPConnection):
|
||||
if isinstance(error, SocketTimeout):
|
||||
raise ConnectTimeoutError(
|
||||
self,
|
||||
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
|
||||
) from e
|
||||
else:
|
||||
# Adding `from e` messes with coverage somehow, so it's omitted.
|
||||
# See #2386.
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {error}"
|
||||
"Connection to %s timed out. (connect timeout=%s)"
|
||||
% (self.host, self.timeout),
|
||||
)
|
||||
else: # Defensive: see https://github.com/urllib3/urllib3/pull/3728#pullrequestreview-3816302703
|
||||
else:
|
||||
raise NewConnectionError(
|
||||
self, "Failed to establish a new connection: %s" % error
|
||||
)
|
||||
else:
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {e}"
|
||||
) from e
|
||||
self, "Failed to establish a new connection: %s" % e
|
||||
)
|
||||
|
||||
except OSError as e: # Defensive: PySocks should catch all these.
|
||||
except SocketError as e: # Defensive: PySocks should catch all these.
|
||||
raise NewConnectionError(
|
||||
self, f"Failed to establish a new connection: {e}"
|
||||
) from e
|
||||
self, "Failed to establish a new connection: %s" % e
|
||||
)
|
||||
|
||||
return conn
|
||||
|
||||
@@ -183,12 +169,12 @@ class SOCKSProxyManager(PoolManager):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
proxy_url: str,
|
||||
username: str | None = None,
|
||||
password: str | None = None,
|
||||
num_pools: int = 10,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
**connection_pool_kw: typing.Any,
|
||||
proxy_url,
|
||||
username=None,
|
||||
password=None,
|
||||
num_pools=10,
|
||||
headers=None,
|
||||
**connection_pool_kw
|
||||
):
|
||||
parsed = parse_url(proxy_url)
|
||||
|
||||
@@ -209,7 +195,7 @@ class SOCKSProxyManager(PoolManager):
|
||||
socks_version = socks.PROXY_TYPE_SOCKS4
|
||||
rdns = True
|
||||
else:
|
||||
raise ValueError(f"Unable to determine SOCKS version from {proxy_url}")
|
||||
raise ValueError("Unable to determine SOCKS version from %s" % proxy_url)
|
||||
|
||||
self.proxy_url = proxy_url
|
||||
|
||||
@@ -223,6 +209,8 @@ class SOCKSProxyManager(PoolManager):
|
||||
}
|
||||
connection_pool_kw["_socks_options"] = socks_options
|
||||
|
||||
super().__init__(num_pools, headers, **connection_pool_kw)
|
||||
super(SOCKSProxyManager, self).__init__(
|
||||
num_pools, headers, **connection_pool_kw
|
||||
)
|
||||
|
||||
self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme
|
||||
|
||||
@@ -1,16 +1,6 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import socket
|
||||
import typing
|
||||
import warnings
|
||||
from email.errors import MessageDefect
|
||||
from http.client import IncompleteRead as httplib_IncompleteRead
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from .connection import HTTPConnection
|
||||
from .connectionpool import ConnectionPool
|
||||
from .response import HTTPResponse
|
||||
from .util.retry import Retry
|
||||
from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead
|
||||
|
||||
# Base Exceptions
|
||||
|
||||
@@ -18,61 +8,64 @@ if typing.TYPE_CHECKING:
|
||||
class HTTPError(Exception):
|
||||
"""Base exception used by this module."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class HTTPWarning(Warning):
|
||||
"""Base warning used by this module."""
|
||||
|
||||
|
||||
_TYPE_REDUCE_RESULT = tuple[typing.Callable[..., object], tuple[object, ...]]
|
||||
pass
|
||||
|
||||
|
||||
class PoolError(HTTPError):
|
||||
"""Base exception for errors caused within a pool."""
|
||||
|
||||
def __init__(self, pool: ConnectionPool, message: str) -> None:
|
||||
def __init__(self, pool, message):
|
||||
self.pool = pool
|
||||
self._message = message
|
||||
super().__init__(f"{pool}: {message}")
|
||||
HTTPError.__init__(self, "%s: %s" % (pool, message))
|
||||
|
||||
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
||||
def __reduce__(self):
|
||||
# For pickling purposes.
|
||||
return self.__class__, (None, self._message)
|
||||
return self.__class__, (None, None)
|
||||
|
||||
|
||||
class RequestError(PoolError):
|
||||
"""Base exception for PoolErrors that have associated URLs."""
|
||||
|
||||
def __init__(self, pool: ConnectionPool, url: str | None, message: str) -> None:
|
||||
def __init__(self, pool, url, message):
|
||||
self.url = url
|
||||
super().__init__(pool, message)
|
||||
PoolError.__init__(self, pool, message)
|
||||
|
||||
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
||||
def __reduce__(self):
|
||||
# For pickling purposes.
|
||||
return self.__class__, (None, self.url, self._message)
|
||||
return self.__class__, (None, self.url, None)
|
||||
|
||||
|
||||
class SSLError(HTTPError):
|
||||
"""Raised when SSL certificate fails in an HTTPS connection."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ProxyError(HTTPError):
|
||||
"""Raised when the connection to a proxy fails."""
|
||||
|
||||
# The original error is also available as __cause__.
|
||||
original_error: Exception
|
||||
|
||||
def __init__(self, message: str, error: Exception) -> None:
|
||||
super().__init__(message, error)
|
||||
def __init__(self, message, error, *args):
|
||||
super(ProxyError, self).__init__(message, error, *args)
|
||||
self.original_error = error
|
||||
|
||||
|
||||
class DecodeError(HTTPError):
|
||||
"""Raised when automatic decoding based on Content-Type fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ProtocolError(HTTPError):
|
||||
"""Raised when something unexpected happens mid-request/response."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
#: Renamed to ProtocolError but aliased for backwards compatibility.
|
||||
ConnectionError = ProtocolError
|
||||
@@ -86,40 +79,33 @@ class MaxRetryError(RequestError):
|
||||
|
||||
:param pool: The connection pool
|
||||
:type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
|
||||
:param str url: The requested Url
|
||||
:param reason: The underlying error
|
||||
:type reason: :class:`Exception`
|
||||
:param string url: The requested Url
|
||||
:param exceptions.Exception reason: The underlying error
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, pool: ConnectionPool, url: str | None, reason: Exception | None = None
|
||||
) -> None:
|
||||
def __init__(self, pool, url, reason=None):
|
||||
self.reason = reason
|
||||
|
||||
message = f"Max retries exceeded with url: {url} (Caused by {reason!r})"
|
||||
message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason)
|
||||
|
||||
super().__init__(pool, url, message)
|
||||
|
||||
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
||||
# For pickling purposes.
|
||||
return self.__class__, (None, self.url, self.reason)
|
||||
RequestError.__init__(self, pool, url, message)
|
||||
|
||||
|
||||
class HostChangedError(RequestError):
|
||||
"""Raised when an existing pool gets a request for a foreign host."""
|
||||
|
||||
def __init__(
|
||||
self, pool: ConnectionPool, url: str, retries: Retry | int = 3
|
||||
) -> None:
|
||||
message = f"Tried to open a foreign host with url: {url}"
|
||||
super().__init__(pool, url, message)
|
||||
def __init__(self, pool, url, retries=3):
|
||||
message = "Tried to open a foreign host with url: %s" % url
|
||||
RequestError.__init__(self, pool, url, message)
|
||||
self.retries = retries
|
||||
|
||||
|
||||
class TimeoutStateError(HTTPError):
|
||||
"""Raised when passing an invalid state to a timeout"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class TimeoutError(HTTPError):
|
||||
"""Raised when a socket timeout error occurs.
|
||||
@@ -128,77 +114,53 @@ class TimeoutError(HTTPError):
|
||||
<ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ReadTimeoutError(TimeoutError, RequestError):
|
||||
"""Raised when a socket timeout occurs while receiving data from a server"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
# This timeout error does not have a URL attached and needs to inherit from the
|
||||
# base HTTPError
|
||||
class ConnectTimeoutError(TimeoutError):
|
||||
"""Raised when a socket timeout occurs while connecting to a server"""
|
||||
|
||||
pass
|
||||
|
||||
class NewConnectionError(ConnectTimeoutError, HTTPError):
|
||||
|
||||
class NewConnectionError(ConnectTimeoutError, PoolError):
|
||||
"""Raised when we fail to establish a new connection. Usually ECONNREFUSED."""
|
||||
|
||||
def __init__(self, conn: HTTPConnection, message: str) -> None:
|
||||
self.conn = conn
|
||||
self._message = message
|
||||
super().__init__(f"{conn}: {message}")
|
||||
|
||||
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
||||
# For pickling purposes.
|
||||
return self.__class__, (None, self._message)
|
||||
|
||||
@property
|
||||
def pool(self) -> HTTPConnection:
|
||||
warnings.warn(
|
||||
"The 'pool' property is deprecated and will be removed "
|
||||
"in urllib3 v3.0. Use 'conn' instead.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
return self.conn
|
||||
|
||||
|
||||
class NameResolutionError(NewConnectionError):
|
||||
"""Raised when host name resolution fails."""
|
||||
|
||||
def __init__(self, host: str, conn: HTTPConnection, reason: socket.gaierror):
|
||||
message = f"Failed to resolve '{host}' ({reason})"
|
||||
self._host = host
|
||||
self._reason = reason
|
||||
super().__init__(conn, message)
|
||||
|
||||
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
|
||||
# For pickling purposes.
|
||||
return self.__class__, (self._host, None, self._reason)
|
||||
pass
|
||||
|
||||
|
||||
class EmptyPoolError(PoolError):
|
||||
"""Raised when a pool runs out of connections and no more are allowed."""
|
||||
|
||||
|
||||
class FullPoolError(PoolError):
|
||||
"""Raised when we try to add a connection to a full pool in blocking mode."""
|
||||
pass
|
||||
|
||||
|
||||
class ClosedPoolError(PoolError):
|
||||
"""Raised when a request enters a pool after the pool has been closed."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class LocationValueError(ValueError, HTTPError):
|
||||
"""Raised when there is something wrong with a given URL input."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class LocationParseError(LocationValueError):
|
||||
"""Raised when get_host or similar fails to parse the URL input."""
|
||||
|
||||
def __init__(self, location: str) -> None:
|
||||
message = f"Failed to parse: {location}"
|
||||
super().__init__(message)
|
||||
def __init__(self, location):
|
||||
message = "Failed to parse: %s" % location
|
||||
HTTPError.__init__(self, message)
|
||||
|
||||
self.location = location
|
||||
|
||||
@@ -206,9 +168,9 @@ class LocationParseError(LocationValueError):
|
||||
class URLSchemeUnknown(LocationValueError):
|
||||
"""Raised when a URL input has an unsupported scheme."""
|
||||
|
||||
def __init__(self, scheme: str):
|
||||
message = f"Not supported URL scheme {scheme}"
|
||||
super().__init__(message)
|
||||
def __init__(self, scheme):
|
||||
message = "Not supported URL scheme %s" % scheme
|
||||
super(URLSchemeUnknown, self).__init__(message)
|
||||
|
||||
self.scheme = scheme
|
||||
|
||||
@@ -223,22 +185,38 @@ class ResponseError(HTTPError):
|
||||
class SecurityWarning(HTTPWarning):
|
||||
"""Warned when performing security reducing actions"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class SubjectAltNameWarning(SecurityWarning):
|
||||
"""Warned when connecting to a host with a certificate missing a SAN."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class InsecureRequestWarning(SecurityWarning):
|
||||
"""Warned when making an unverified HTTPS request."""
|
||||
|
||||
|
||||
class NotOpenSSLWarning(SecurityWarning):
|
||||
"""Warned when using unsupported SSL library"""
|
||||
pass
|
||||
|
||||
|
||||
class SystemTimeWarning(SecurityWarning):
|
||||
"""Warned when system time is suspected to be wrong"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class InsecurePlatformWarning(SecurityWarning):
|
||||
"""Warned when certain TLS/SSL configuration is not available on a platform."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class SNIMissingWarning(HTTPWarning):
|
||||
"""Warned when making a HTTPS request without SNI available."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DependencyWarning(HTTPWarning):
|
||||
"""
|
||||
@@ -246,10 +224,14 @@ class DependencyWarning(HTTPWarning):
|
||||
dependencies.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ResponseNotChunked(ProtocolError, ValueError):
|
||||
"""Response needs to be chunked in order to read it as chunks."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class BodyNotHttplibCompatible(HTTPError):
|
||||
"""
|
||||
@@ -257,6 +239,8 @@ class BodyNotHttplibCompatible(HTTPError):
|
||||
(have an fp attribute which returns raw chunks) for read_chunked().
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class IncompleteRead(HTTPError, httplib_IncompleteRead):
|
||||
"""
|
||||
@@ -266,14 +250,10 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead):
|
||||
for ``partial`` to avoid creating large objects on streamed reads.
|
||||
"""
|
||||
|
||||
partial: int # type: ignore[assignment]
|
||||
expected: int
|
||||
def __init__(self, partial, expected):
|
||||
super(IncompleteRead, self).__init__(partial, expected)
|
||||
|
||||
def __init__(self, partial: int, expected: int) -> None:
|
||||
self.partial = partial
|
||||
self.expected = expected
|
||||
|
||||
def __repr__(self) -> str:
|
||||
def __repr__(self):
|
||||
return "IncompleteRead(%i bytes read, %i more expected)" % (
|
||||
self.partial,
|
||||
self.expected,
|
||||
@@ -283,13 +263,14 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead):
|
||||
class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
|
||||
"""Invalid chunk length in a chunked response."""
|
||||
|
||||
def __init__(self, response: HTTPResponse, length: bytes) -> None:
|
||||
self.partial: int = response.tell() # type: ignore[assignment]
|
||||
self.expected: int | None = response.length_remaining
|
||||
def __init__(self, response, length):
|
||||
super(InvalidChunkLength, self).__init__(
|
||||
response.tell(), response.length_remaining
|
||||
)
|
||||
self.response = response
|
||||
self.length = length
|
||||
|
||||
def __repr__(self) -> str:
|
||||
def __repr__(self):
|
||||
return "InvalidChunkLength(got length %r, %i bytes read)" % (
|
||||
self.length,
|
||||
self.partial,
|
||||
@@ -299,13 +280,15 @@ class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
|
||||
class InvalidHeader(HTTPError):
|
||||
"""The header provided was somehow invalid."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ProxySchemeUnknown(AssertionError, URLSchemeUnknown):
|
||||
"""ProxyManager does not support the supplied scheme"""
|
||||
|
||||
# TODO(t-8ch): Stop inheriting from AssertionError in v2.0.
|
||||
|
||||
def __init__(self, scheme: str | None) -> None:
|
||||
def __init__(self, scheme):
|
||||
# 'localhost' is here because our URL parser parses
|
||||
# localhost:8080 -> scheme=localhost, remove if we fix this.
|
||||
if scheme == "localhost":
|
||||
@@ -313,23 +296,28 @@ class ProxySchemeUnknown(AssertionError, URLSchemeUnknown):
|
||||
if scheme is None:
|
||||
message = "Proxy URL had no scheme, should start with http:// or https://"
|
||||
else:
|
||||
message = f"Proxy URL had unsupported scheme {scheme}, should use http:// or https://"
|
||||
super().__init__(message)
|
||||
message = (
|
||||
"Proxy URL had unsupported scheme %s, should use http:// or https://"
|
||||
% scheme
|
||||
)
|
||||
super(ProxySchemeUnknown, self).__init__(message)
|
||||
|
||||
|
||||
class ProxySchemeUnsupported(ValueError):
|
||||
"""Fetching HTTPS resources through HTTPS proxies is unsupported"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class HeaderParsingError(HTTPError):
|
||||
"""Raised by assert_header_parsing, but we convert it to a log.warning statement."""
|
||||
|
||||
def __init__(
|
||||
self, defects: list[MessageDefect], unparsed_data: bytes | str | None
|
||||
) -> None:
|
||||
message = f"{defects or 'Unknown'}, unparsed data: {unparsed_data!r}"
|
||||
super().__init__(message)
|
||||
def __init__(self, defects, unparsed_data):
|
||||
message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data)
|
||||
super(HeaderParsingError, self).__init__(message)
|
||||
|
||||
|
||||
class UnrewindableBodyError(HTTPError):
|
||||
"""urllib3 encountered an error when trying to rewind a body"""
|
||||
|
||||
pass
|
||||
|
||||
@@ -1,20 +1,13 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import email.utils
|
||||
import mimetypes
|
||||
import typing
|
||||
import re
|
||||
|
||||
_TYPE_FIELD_VALUE = typing.Union[str, bytes]
|
||||
_TYPE_FIELD_VALUE_TUPLE = typing.Union[
|
||||
_TYPE_FIELD_VALUE,
|
||||
tuple[str, _TYPE_FIELD_VALUE],
|
||||
tuple[str, _TYPE_FIELD_VALUE, str],
|
||||
]
|
||||
from .packages import six
|
||||
|
||||
|
||||
def guess_content_type(
|
||||
filename: str | None, default: str = "application/octet-stream"
|
||||
) -> str:
|
||||
def guess_content_type(filename, default="application/octet-stream"):
|
||||
"""
|
||||
Guess the "Content-Type" of a file.
|
||||
|
||||
@@ -28,7 +21,7 @@ def guess_content_type(
|
||||
return default
|
||||
|
||||
|
||||
def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
def format_header_param_rfc2231(name, value):
|
||||
"""
|
||||
Helper function to format and quote a single header parameter using the
|
||||
strategy defined in RFC 2231.
|
||||
@@ -41,28 +34,14 @@ def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
The name of the parameter, a string expected to be ASCII only.
|
||||
:param value:
|
||||
The value of the parameter, provided as ``bytes`` or `str``.
|
||||
:returns:
|
||||
:ret:
|
||||
An RFC-2231-formatted unicode string.
|
||||
|
||||
.. deprecated:: 2.0.0
|
||||
Will be removed in urllib3 v3.0. This is not valid for
|
||||
``multipart/form-data`` header parameters.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"'format_header_param_rfc2231' is insecure, deprecated and will be "
|
||||
"removed in urllib3 v3.0. This is not valid for "
|
||||
"multipart/form-data header parameters.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if isinstance(value, bytes):
|
||||
if isinstance(value, six.binary_type):
|
||||
value = value.decode("utf-8")
|
||||
|
||||
if not any(ch in value for ch in '"\\\r\n'):
|
||||
result = f'{name}="{value}"'
|
||||
result = u'%s="%s"' % (name, value)
|
||||
try:
|
||||
result.encode("ascii")
|
||||
except (UnicodeEncodeError, UnicodeDecodeError):
|
||||
@@ -70,87 +49,81 @@ def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
else:
|
||||
return result
|
||||
|
||||
if six.PY2: # Python 2:
|
||||
value = value.encode("utf-8")
|
||||
|
||||
# encode_rfc2231 accepts an encoded string and returns an ascii-encoded
|
||||
# string in Python 2 but accepts and returns unicode strings in Python 3
|
||||
value = email.utils.encode_rfc2231(value, "utf-8")
|
||||
value = f"{name}*={value}"
|
||||
value = "%s*=%s" % (name, value)
|
||||
|
||||
if six.PY2: # Python 2:
|
||||
value = value.decode("utf-8")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def format_multipart_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
_HTML5_REPLACEMENTS = {
|
||||
u"\u0022": u"%22",
|
||||
# Replace "\" with "\\".
|
||||
u"\u005C": u"\u005C\u005C",
|
||||
}
|
||||
|
||||
# All control characters from 0x00 to 0x1F *except* 0x1B.
|
||||
_HTML5_REPLACEMENTS.update(
|
||||
{
|
||||
six.unichr(cc): u"%{:02X}".format(cc)
|
||||
for cc in range(0x00, 0x1F + 1)
|
||||
if cc not in (0x1B,)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _replace_multiple(value, needles_and_replacements):
|
||||
def replacer(match):
|
||||
return needles_and_replacements[match.group(0)]
|
||||
|
||||
pattern = re.compile(
|
||||
r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()])
|
||||
)
|
||||
|
||||
result = pattern.sub(replacer, value)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def format_header_param_html5(name, value):
|
||||
"""
|
||||
Format and quote a single multipart header parameter.
|
||||
Helper function to format and quote a single header parameter using the
|
||||
HTML5 strategy.
|
||||
|
||||
This follows the `WHATWG HTML Standard`_ as of 2021/06/10, matching
|
||||
the behavior of current browser and curl versions. Values are
|
||||
assumed to be UTF-8. The ``\\n``, ``\\r``, and ``"`` characters are
|
||||
percent encoded.
|
||||
Particularly useful for header parameters which might contain
|
||||
non-ASCII values, like file names. This follows the `HTML5 Working Draft
|
||||
Section 4.10.22.7`_ and matches the behavior of curl and modern browsers.
|
||||
|
||||
.. _WHATWG HTML Standard:
|
||||
https://html.spec.whatwg.org/multipage/
|
||||
form-control-infrastructure.html#multipart-form-data
|
||||
.. _HTML5 Working Draft Section 4.10.22.7:
|
||||
https://w3c.github.io/html/sec-forms.html#multipart-form-data
|
||||
|
||||
:param name:
|
||||
The name of the parameter, an ASCII-only ``str``.
|
||||
The name of the parameter, a string expected to be ASCII only.
|
||||
:param value:
|
||||
The value of the parameter, a ``str`` or UTF-8 encoded
|
||||
``bytes``.
|
||||
:returns:
|
||||
A string ``name="value"`` with the escaped value.
|
||||
|
||||
.. versionchanged:: 2.0.0
|
||||
Matches the WHATWG HTML Standard as of 2021/06/10. Control
|
||||
characters are no longer percent encoded.
|
||||
|
||||
.. versionchanged:: 2.0.0
|
||||
Renamed from ``format_header_param_html5`` and
|
||||
``format_header_param``. The old names will be removed in
|
||||
urllib3 v3.0.
|
||||
The value of the parameter, provided as ``bytes`` or `str``.
|
||||
:ret:
|
||||
A unicode string, stripped of troublesome characters.
|
||||
"""
|
||||
if isinstance(value, bytes):
|
||||
if isinstance(value, six.binary_type):
|
||||
value = value.decode("utf-8")
|
||||
|
||||
# percent encode \n \r "
|
||||
value = value.translate({10: "%0A", 13: "%0D", 34: "%22"})
|
||||
return f'{name}="{value}"'
|
||||
value = _replace_multiple(value, _HTML5_REPLACEMENTS)
|
||||
|
||||
return u'%s="%s"' % (name, value)
|
||||
|
||||
|
||||
def format_header_param_html5(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
"""
|
||||
.. deprecated:: 2.0.0
|
||||
Renamed to :func:`format_multipart_header_param`. Will be
|
||||
removed in urllib3 v3.0.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"'format_header_param_html5' has been renamed to "
|
||||
"'format_multipart_header_param'. The old name will be "
|
||||
"removed in urllib3 v3.0.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return format_multipart_header_param(name, value)
|
||||
# For backwards-compatibility.
|
||||
format_header_param = format_header_param_html5
|
||||
|
||||
|
||||
def format_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
"""
|
||||
.. deprecated:: 2.0.0
|
||||
Renamed to :func:`format_multipart_header_param`. Will be
|
||||
removed in urllib3 v3.0.
|
||||
"""
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"'format_header_param' has been renamed to "
|
||||
"'format_multipart_header_param'. The old name will be "
|
||||
"removed in urllib3 v3.0.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return format_multipart_header_param(name, value)
|
||||
|
||||
|
||||
class RequestField:
|
||||
class RequestField(object):
|
||||
"""
|
||||
A data container for request body parameters.
|
||||
|
||||
@@ -162,47 +135,29 @@ class RequestField:
|
||||
An optional filename of the request field. Must be unicode.
|
||||
:param headers:
|
||||
An optional dict-like object of headers to initially use for the field.
|
||||
|
||||
.. versionchanged:: 2.0.0
|
||||
The ``header_formatter`` parameter is deprecated and will
|
||||
be removed in urllib3 v3.0.
|
||||
:param header_formatter:
|
||||
An optional callable that is used to encode and format the headers. By
|
||||
default, this is :func:`format_header_param_html5`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
data: _TYPE_FIELD_VALUE,
|
||||
filename: str | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
|
||||
name,
|
||||
data,
|
||||
filename=None,
|
||||
headers=None,
|
||||
header_formatter=format_header_param_html5,
|
||||
):
|
||||
self._name = name
|
||||
self._filename = filename
|
||||
self.data = data
|
||||
self.headers: dict[str, str | None] = {}
|
||||
self.headers = {}
|
||||
if headers:
|
||||
self.headers = dict(headers)
|
||||
|
||||
if header_formatter is not None:
|
||||
import warnings
|
||||
|
||||
warnings.warn(
|
||||
"The 'header_formatter' parameter is deprecated and "
|
||||
"will be removed in urllib3 v3.0.",
|
||||
FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self.header_formatter = header_formatter
|
||||
else:
|
||||
self.header_formatter = format_multipart_header_param
|
||||
self.header_formatter = header_formatter
|
||||
|
||||
@classmethod
|
||||
def from_tuples(
|
||||
cls,
|
||||
fieldname: str,
|
||||
value: _TYPE_FIELD_VALUE_TUPLE,
|
||||
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
|
||||
) -> RequestField:
|
||||
def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5):
|
||||
"""
|
||||
A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
|
||||
|
||||
@@ -219,10 +174,6 @@ class RequestField:
|
||||
|
||||
Field names and filenames must be unicode.
|
||||
"""
|
||||
filename: str | None
|
||||
content_type: str | None
|
||||
data: _TYPE_FIELD_VALUE
|
||||
|
||||
if isinstance(value, tuple):
|
||||
if len(value) == 3:
|
||||
filename, data, content_type = value
|
||||
@@ -241,29 +192,20 @@ class RequestField:
|
||||
|
||||
return request_param
|
||||
|
||||
def _render_part(self, name: str, value: _TYPE_FIELD_VALUE) -> str:
|
||||
def _render_part(self, name, value):
|
||||
"""
|
||||
Override this method to change how each multipart header
|
||||
parameter is formatted. By default, this calls
|
||||
:func:`format_multipart_header_param`.
|
||||
Overridable helper function to format a single header parameter. By
|
||||
default, this calls ``self.header_formatter``.
|
||||
|
||||
:param name:
|
||||
The name of the parameter, an ASCII-only ``str``.
|
||||
The name of the parameter, a string expected to be ASCII only.
|
||||
:param value:
|
||||
The value of the parameter, a ``str`` or UTF-8 encoded
|
||||
``bytes``.
|
||||
|
||||
:meta public:
|
||||
The value of the parameter, provided as a unicode string.
|
||||
"""
|
||||
|
||||
return self.header_formatter(name, value)
|
||||
|
||||
def _render_parts(
|
||||
self,
|
||||
header_parts: (
|
||||
dict[str, _TYPE_FIELD_VALUE | None]
|
||||
| typing.Sequence[tuple[str, _TYPE_FIELD_VALUE | None]]
|
||||
),
|
||||
) -> str:
|
||||
def _render_parts(self, header_parts):
|
||||
"""
|
||||
Helper function to format and quote a single header.
|
||||
|
||||
@@ -274,21 +216,18 @@ class RequestField:
|
||||
A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format
|
||||
as `k1="v1"; k2="v2"; ...`.
|
||||
"""
|
||||
iterable: typing.Iterable[tuple[str, _TYPE_FIELD_VALUE | None]]
|
||||
|
||||
parts = []
|
||||
iterable = header_parts
|
||||
if isinstance(header_parts, dict):
|
||||
iterable = header_parts.items()
|
||||
else:
|
||||
iterable = header_parts
|
||||
|
||||
for name, value in iterable:
|
||||
if value is not None:
|
||||
parts.append(self._render_part(name, value))
|
||||
|
||||
return "; ".join(parts)
|
||||
return u"; ".join(parts)
|
||||
|
||||
def render_headers(self) -> str:
|
||||
def render_headers(self):
|
||||
"""
|
||||
Renders the headers for this request field.
|
||||
"""
|
||||
@@ -297,45 +236,39 @@ class RequestField:
|
||||
sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
|
||||
for sort_key in sort_keys:
|
||||
if self.headers.get(sort_key, False):
|
||||
lines.append(f"{sort_key}: {self.headers[sort_key]}")
|
||||
lines.append(u"%s: %s" % (sort_key, self.headers[sort_key]))
|
||||
|
||||
for header_name, header_value in self.headers.items():
|
||||
if header_name not in sort_keys:
|
||||
if header_value:
|
||||
lines.append(f"{header_name}: {header_value}")
|
||||
lines.append(u"%s: %s" % (header_name, header_value))
|
||||
|
||||
lines.append("\r\n")
|
||||
return "\r\n".join(lines)
|
||||
lines.append(u"\r\n")
|
||||
return u"\r\n".join(lines)
|
||||
|
||||
def make_multipart(
|
||||
self,
|
||||
content_disposition: str | None = None,
|
||||
content_type: str | None = None,
|
||||
content_location: str | None = None,
|
||||
) -> None:
|
||||
self, content_disposition=None, content_type=None, content_location=None
|
||||
):
|
||||
"""
|
||||
Makes this request field into a multipart request field.
|
||||
|
||||
This method overrides "Content-Disposition", "Content-Type" and
|
||||
"Content-Location" headers to the request parameter.
|
||||
|
||||
:param content_disposition:
|
||||
The 'Content-Disposition' of the request body. Defaults to 'form-data'
|
||||
:param content_type:
|
||||
The 'Content-Type' of the request body.
|
||||
:param content_location:
|
||||
The 'Content-Location' of the request body.
|
||||
|
||||
"""
|
||||
content_disposition = (content_disposition or "form-data") + "; ".join(
|
||||
self.headers["Content-Disposition"] = content_disposition or u"form-data"
|
||||
self.headers["Content-Disposition"] += u"; ".join(
|
||||
[
|
||||
"",
|
||||
u"",
|
||||
self._render_parts(
|
||||
(("name", self._name), ("filename", self._filename))
|
||||
((u"name", self._name), (u"filename", self._filename))
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
self.headers["Content-Disposition"] = content_disposition
|
||||
self.headers["Content-Type"] = content_type
|
||||
self.headers["Content-Location"] = content_location
|
||||
|
||||
@@ -1,32 +1,28 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import binascii
|
||||
import codecs
|
||||
import os
|
||||
import typing
|
||||
from io import BytesIO
|
||||
|
||||
from .fields import _TYPE_FIELD_VALUE_TUPLE, RequestField
|
||||
from .fields import RequestField
|
||||
from .packages import six
|
||||
from .packages.six import b
|
||||
|
||||
writer = codecs.lookup("utf-8")[3]
|
||||
|
||||
_TYPE_FIELDS_SEQUENCE = typing.Sequence[
|
||||
typing.Union[tuple[str, _TYPE_FIELD_VALUE_TUPLE], RequestField]
|
||||
]
|
||||
_TYPE_FIELDS = typing.Union[
|
||||
_TYPE_FIELDS_SEQUENCE,
|
||||
typing.Mapping[str, _TYPE_FIELD_VALUE_TUPLE],
|
||||
]
|
||||
|
||||
|
||||
def choose_boundary() -> str:
|
||||
def choose_boundary():
|
||||
"""
|
||||
Our embarrassingly-simple replacement for mimetools.choose_boundary.
|
||||
"""
|
||||
return binascii.hexlify(os.urandom(16)).decode()
|
||||
boundary = binascii.hexlify(os.urandom(16))
|
||||
if not six.PY2:
|
||||
boundary = boundary.decode("ascii")
|
||||
return boundary
|
||||
|
||||
|
||||
def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]:
|
||||
def iter_field_objects(fields):
|
||||
"""
|
||||
Iterate over fields.
|
||||
|
||||
@@ -34,29 +30,42 @@ def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]:
|
||||
:class:`~urllib3.fields.RequestField`.
|
||||
|
||||
"""
|
||||
iterable: typing.Iterable[RequestField | tuple[str, _TYPE_FIELD_VALUE_TUPLE]]
|
||||
|
||||
if isinstance(fields, typing.Mapping):
|
||||
iterable = fields.items()
|
||||
if isinstance(fields, dict):
|
||||
i = six.iteritems(fields)
|
||||
else:
|
||||
iterable = fields
|
||||
i = iter(fields)
|
||||
|
||||
for field in iterable:
|
||||
for field in i:
|
||||
if isinstance(field, RequestField):
|
||||
yield field
|
||||
else:
|
||||
yield RequestField.from_tuples(*field)
|
||||
|
||||
|
||||
def encode_multipart_formdata(
|
||||
fields: _TYPE_FIELDS, boundary: str | None = None
|
||||
) -> tuple[bytes, str]:
|
||||
def iter_fields(fields):
|
||||
"""
|
||||
.. deprecated:: 1.6
|
||||
|
||||
Iterate over fields.
|
||||
|
||||
The addition of :class:`~urllib3.fields.RequestField` makes this function
|
||||
obsolete. Instead, use :func:`iter_field_objects`, which returns
|
||||
:class:`~urllib3.fields.RequestField` objects.
|
||||
|
||||
Supports list of (k, v) tuples and dicts.
|
||||
"""
|
||||
if isinstance(fields, dict):
|
||||
return ((k, v) for k, v in six.iteritems(fields))
|
||||
|
||||
return ((k, v) for k, v in fields)
|
||||
|
||||
|
||||
def encode_multipart_formdata(fields, boundary=None):
|
||||
"""
|
||||
Encode a dictionary of ``fields`` using the multipart/form-data MIME format.
|
||||
|
||||
:param fields:
|
||||
Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`).
|
||||
Values are processed by :func:`urllib3.fields.RequestField.from_tuples`.
|
||||
|
||||
:param boundary:
|
||||
If not specified, then a random boundary will be generated using
|
||||
@@ -67,7 +76,7 @@ def encode_multipart_formdata(
|
||||
boundary = choose_boundary()
|
||||
|
||||
for field in iter_field_objects(fields):
|
||||
body.write(f"--{boundary}\r\n".encode("latin-1"))
|
||||
body.write(b("--%s\r\n" % (boundary)))
|
||||
|
||||
writer(body).write(field.render_headers())
|
||||
data = field.data
|
||||
@@ -75,15 +84,15 @@ def encode_multipart_formdata(
|
||||
if isinstance(data, int):
|
||||
data = str(data) # Backwards compatibility
|
||||
|
||||
if isinstance(data, str):
|
||||
if isinstance(data, six.text_type):
|
||||
writer(body).write(data)
|
||||
else:
|
||||
body.write(data)
|
||||
|
||||
body.write(b"\r\n")
|
||||
|
||||
body.write(f"--{boundary}--\r\n".encode("latin-1"))
|
||||
body.write(b("--%s--\r\n" % (boundary)))
|
||||
|
||||
content_type = f"multipart/form-data; boundary={boundary}"
|
||||
content_type = str("multipart/form-data; boundary=%s" % boundary)
|
||||
|
||||
return body.getvalue(), content_type
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib.metadata import version
|
||||
|
||||
__all__ = [
|
||||
"inject_into_urllib3",
|
||||
"extract_from_urllib3",
|
||||
]
|
||||
|
||||
import typing
|
||||
|
||||
orig_HTTPSConnection: typing.Any = None
|
||||
|
||||
|
||||
def inject_into_urllib3() -> None:
|
||||
# First check if h2 version is valid
|
||||
h2_version = version("h2")
|
||||
if not h2_version.startswith("4."):
|
||||
raise ImportError(
|
||||
"urllib3 v2 supports h2 version 4.x.x, currently "
|
||||
f"the 'h2' module is compiled with {h2_version!r}. "
|
||||
"See: https://github.com/urllib3/urllib3/issues/3290"
|
||||
)
|
||||
|
||||
# Import here to avoid circular dependencies.
|
||||
from .. import connection as urllib3_connection
|
||||
from .. import util as urllib3_util
|
||||
from ..connectionpool import HTTPSConnectionPool
|
||||
from ..util import ssl_ as urllib3_util_ssl
|
||||
from .connection import HTTP2Connection
|
||||
|
||||
global orig_HTTPSConnection
|
||||
orig_HTTPSConnection = urllib3_connection.HTTPSConnection
|
||||
|
||||
HTTPSConnectionPool.ConnectionCls = HTTP2Connection
|
||||
urllib3_connection.HTTPSConnection = HTTP2Connection # type: ignore[misc]
|
||||
|
||||
# TODO: Offer 'http/1.1' as well, but for testing purposes this is handy.
|
||||
urllib3_util.ALPN_PROTOCOLS = ["h2"]
|
||||
urllib3_util_ssl.ALPN_PROTOCOLS = ["h2"]
|
||||
|
||||
|
||||
def extract_from_urllib3() -> None:
|
||||
from .. import connection as urllib3_connection
|
||||
from .. import util as urllib3_util
|
||||
from ..connectionpool import HTTPSConnectionPool
|
||||
from ..util import ssl_ as urllib3_util_ssl
|
||||
|
||||
HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection
|
||||
urllib3_connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc]
|
||||
|
||||
urllib3_util.ALPN_PROTOCOLS = ["http/1.1"]
|
||||
urllib3_util_ssl.ALPN_PROTOCOLS = ["http/1.1"]
|
||||
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -1,356 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import types
|
||||
import typing
|
||||
|
||||
import h2.config
|
||||
import h2.connection
|
||||
import h2.events
|
||||
|
||||
from .._base_connection import _TYPE_BODY
|
||||
from .._collections import HTTPHeaderDict
|
||||
from ..connection import HTTPSConnection, _get_default_user_agent
|
||||
from ..exceptions import ConnectionError
|
||||
from ..response import BaseHTTPResponse
|
||||
|
||||
orig_HTTPSConnection = HTTPSConnection
|
||||
|
||||
T = typing.TypeVar("T")
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
RE_IS_LEGAL_HEADER_NAME = re.compile(rb"^[!#$%&'*+\-.^_`|~0-9a-z]+$")
|
||||
RE_IS_ILLEGAL_HEADER_VALUE = re.compile(rb"[\0\x00\x0a\x0d\r\n]|^[ \r\n\t]|[ \r\n\t]$")
|
||||
|
||||
|
||||
def _is_legal_header_name(name: bytes) -> bool:
|
||||
"""
|
||||
"An implementation that validates fields according to the definitions in Sections
|
||||
5.1 and 5.5 of [HTTP] only needs an additional check that field names do not
|
||||
include uppercase characters." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
|
||||
|
||||
`http.client._is_legal_header_name` does not validate the field name according to the
|
||||
HTTP 1.1 spec, so we do that here, in addition to checking for uppercase characters.
|
||||
|
||||
This does not allow for the `:` character in the header name, so should not
|
||||
be used to validate pseudo-headers.
|
||||
"""
|
||||
return bool(RE_IS_LEGAL_HEADER_NAME.match(name))
|
||||
|
||||
|
||||
def _is_illegal_header_value(value: bytes) -> bool:
|
||||
"""
|
||||
"A field value MUST NOT contain the zero value (ASCII NUL, 0x00), line feed
|
||||
(ASCII LF, 0x0a), or carriage return (ASCII CR, 0x0d) at any position. A field
|
||||
value MUST NOT start or end with an ASCII whitespace character (ASCII SP or HTAB,
|
||||
0x20 or 0x09)." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
|
||||
"""
|
||||
return bool(RE_IS_ILLEGAL_HEADER_VALUE.search(value))
|
||||
|
||||
|
||||
class _LockedObject(typing.Generic[T]):
|
||||
"""
|
||||
A wrapper class that hides a specific object behind a lock.
|
||||
The goal here is to provide a simple way to protect access to an object
|
||||
that cannot safely be simultaneously accessed from multiple threads. The
|
||||
intended use of this class is simple: take hold of it with a context
|
||||
manager, which returns the protected object.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"lock",
|
||||
"_obj",
|
||||
)
|
||||
|
||||
def __init__(self, obj: T):
|
||||
self.lock = threading.RLock()
|
||||
self._obj = obj
|
||||
|
||||
def __enter__(self) -> T:
|
||||
self.lock.acquire()
|
||||
return self._obj
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_val: BaseException | None,
|
||||
exc_tb: types.TracebackType | None,
|
||||
) -> None:
|
||||
self.lock.release()
|
||||
|
||||
|
||||
class HTTP2Connection(HTTPSConnection):
|
||||
def __init__(
|
||||
self, host: str, port: int | None = None, **kwargs: typing.Any
|
||||
) -> None:
|
||||
self._h2_conn = self._new_h2_conn()
|
||||
self._h2_stream: int | None = None
|
||||
self._headers: list[tuple[bytes, bytes]] = []
|
||||
|
||||
if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive:
|
||||
raise NotImplementedError("Proxies aren't supported with HTTP/2")
|
||||
|
||||
super().__init__(host, port, **kwargs)
|
||||
|
||||
if self._tunnel_host is not None:
|
||||
raise NotImplementedError("Tunneling isn't supported with HTTP/2")
|
||||
|
||||
def _new_h2_conn(self) -> _LockedObject[h2.connection.H2Connection]:
|
||||
config = h2.config.H2Configuration(client_side=True)
|
||||
return _LockedObject(h2.connection.H2Connection(config=config))
|
||||
|
||||
def connect(self) -> None:
|
||||
super().connect()
|
||||
with self._h2_conn as conn:
|
||||
conn.initiate_connection()
|
||||
if data_to_send := conn.data_to_send():
|
||||
self.sock.sendall(data_to_send)
|
||||
|
||||
def putrequest( # type: ignore[override]
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
**kwargs: typing.Any,
|
||||
) -> None:
|
||||
"""putrequest
|
||||
This deviates from the HTTPConnection method signature since we never need to override
|
||||
sending accept-encoding headers or the host header.
|
||||
"""
|
||||
if "skip_host" in kwargs:
|
||||
raise NotImplementedError("`skip_host` isn't supported")
|
||||
if "skip_accept_encoding" in kwargs:
|
||||
raise NotImplementedError("`skip_accept_encoding` isn't supported")
|
||||
|
||||
self._request_url = url or "/"
|
||||
self._validate_path(url) # type: ignore[attr-defined]
|
||||
|
||||
if ":" in self.host:
|
||||
authority = f"[{self.host}]:{self.port or 443}"
|
||||
else:
|
||||
authority = f"{self.host}:{self.port or 443}"
|
||||
|
||||
self._headers.append((b":scheme", b"https"))
|
||||
self._headers.append((b":method", method.encode()))
|
||||
self._headers.append((b":authority", authority.encode()))
|
||||
self._headers.append((b":path", url.encode()))
|
||||
|
||||
with self._h2_conn as conn:
|
||||
self._h2_stream = conn.get_next_available_stream_id()
|
||||
|
||||
def putheader(self, header: str | bytes, *values: str | bytes) -> None: # type: ignore[override]
|
||||
# TODO SKIPPABLE_HEADERS from urllib3 are ignored.
|
||||
header = header.encode() if isinstance(header, str) else header
|
||||
header = header.lower() # A lot of upstream code uses capitalized headers.
|
||||
if not _is_legal_header_name(header):
|
||||
raise ValueError(f"Illegal header name {str(header)}")
|
||||
|
||||
for value in values:
|
||||
value = value.encode() if isinstance(value, str) else value
|
||||
if _is_illegal_header_value(value):
|
||||
raise ValueError(f"Illegal header value {str(value)}")
|
||||
self._headers.append((header, value))
|
||||
|
||||
def endheaders(self, message_body: typing.Any = None) -> None: # type: ignore[override]
|
||||
if self._h2_stream is None:
|
||||
raise ConnectionError("Must call `putrequest` first.")
|
||||
|
||||
with self._h2_conn as conn:
|
||||
conn.send_headers(
|
||||
stream_id=self._h2_stream,
|
||||
headers=self._headers,
|
||||
end_stream=(message_body is None),
|
||||
)
|
||||
if data_to_send := conn.data_to_send():
|
||||
self.sock.sendall(data_to_send)
|
||||
self._headers = [] # Reset headers for the next request.
|
||||
|
||||
def send(self, data: typing.Any) -> None:
|
||||
"""Send data to the server.
|
||||
`data` can be: `str`, `bytes`, an iterable, or file-like objects
|
||||
that support a .read() method.
|
||||
"""
|
||||
if self._h2_stream is None:
|
||||
raise ConnectionError("Must call `putrequest` first.")
|
||||
|
||||
with self._h2_conn as conn:
|
||||
if data_to_send := conn.data_to_send():
|
||||
self.sock.sendall(data_to_send)
|
||||
|
||||
if hasattr(data, "read"): # file-like objects
|
||||
while True:
|
||||
chunk = data.read(self.blocksize)
|
||||
if not chunk:
|
||||
break
|
||||
if isinstance(chunk, str):
|
||||
chunk = chunk.encode()
|
||||
conn.send_data(self._h2_stream, chunk, end_stream=False)
|
||||
if data_to_send := conn.data_to_send():
|
||||
self.sock.sendall(data_to_send)
|
||||
conn.end_stream(self._h2_stream)
|
||||
return
|
||||
|
||||
if isinstance(data, str): # str -> bytes
|
||||
data = data.encode()
|
||||
|
||||
try:
|
||||
if isinstance(data, bytes):
|
||||
conn.send_data(self._h2_stream, data, end_stream=True)
|
||||
if data_to_send := conn.data_to_send():
|
||||
self.sock.sendall(data_to_send)
|
||||
else:
|
||||
for chunk in data:
|
||||
conn.send_data(self._h2_stream, chunk, end_stream=False)
|
||||
if data_to_send := conn.data_to_send():
|
||||
self.sock.sendall(data_to_send)
|
||||
conn.end_stream(self._h2_stream)
|
||||
except TypeError:
|
||||
raise TypeError(
|
||||
"`data` should be str, bytes, iterable, or file. got %r"
|
||||
% type(data)
|
||||
)
|
||||
|
||||
def set_tunnel(
|
||||
self,
|
||||
host: str,
|
||||
port: int | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
scheme: str = "http",
|
||||
) -> None:
|
||||
raise NotImplementedError(
|
||||
"HTTP/2 does not support setting up a tunnel through a proxy"
|
||||
)
|
||||
|
||||
def getresponse( # type: ignore[override]
|
||||
self,
|
||||
) -> HTTP2Response:
|
||||
status = None
|
||||
data = bytearray()
|
||||
with self._h2_conn as conn:
|
||||
end_stream = False
|
||||
while not end_stream:
|
||||
# TODO: Arbitrary read value.
|
||||
if received_data := self.sock.recv(65535):
|
||||
events = conn.receive_data(received_data)
|
||||
for event in events:
|
||||
if isinstance(event, h2.events.ResponseReceived):
|
||||
headers = HTTPHeaderDict()
|
||||
for header, value in event.headers:
|
||||
if header == b":status":
|
||||
status = int(value.decode())
|
||||
else:
|
||||
headers.add(
|
||||
header.decode("ascii"), value.decode("ascii")
|
||||
)
|
||||
|
||||
elif isinstance(event, h2.events.DataReceived):
|
||||
data += event.data
|
||||
conn.acknowledge_received_data(
|
||||
event.flow_controlled_length, event.stream_id
|
||||
)
|
||||
|
||||
elif isinstance(event, h2.events.StreamEnded):
|
||||
end_stream = True
|
||||
|
||||
if data_to_send := conn.data_to_send():
|
||||
self.sock.sendall(data_to_send)
|
||||
|
||||
assert status is not None
|
||||
return HTTP2Response(
|
||||
status=status,
|
||||
headers=headers,
|
||||
request_url=self._request_url,
|
||||
data=bytes(data),
|
||||
)
|
||||
|
||||
def request( # type: ignore[override]
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: _TYPE_BODY | None = None,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
*,
|
||||
preload_content: bool = True,
|
||||
decode_content: bool = True,
|
||||
enforce_content_length: bool = True,
|
||||
**kwargs: typing.Any,
|
||||
) -> None:
|
||||
"""Send an HTTP/2 request"""
|
||||
if "chunked" in kwargs:
|
||||
# TODO this is often present from upstream.
|
||||
# raise NotImplementedError("`chunked` isn't supported with HTTP/2")
|
||||
pass
|
||||
|
||||
if self.sock is not None:
|
||||
self.sock.settimeout(self.timeout)
|
||||
|
||||
self.putrequest(method, url)
|
||||
|
||||
headers = headers or {}
|
||||
for k, v in headers.items():
|
||||
if k.lower() == "transfer-encoding" and v == "chunked":
|
||||
continue
|
||||
else:
|
||||
self.putheader(k, v)
|
||||
|
||||
if b"user-agent" not in dict(self._headers):
|
||||
self.putheader(b"user-agent", _get_default_user_agent())
|
||||
|
||||
if body:
|
||||
self.endheaders(message_body=body)
|
||||
self.send(body)
|
||||
else:
|
||||
self.endheaders()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._h2_conn as conn:
|
||||
try:
|
||||
conn.close_connection()
|
||||
if data := conn.data_to_send():
|
||||
self.sock.sendall(data)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Reset all our HTTP/2 connection state.
|
||||
self._h2_conn = self._new_h2_conn()
|
||||
self._h2_stream = None
|
||||
self._headers = []
|
||||
|
||||
super().close()
|
||||
|
||||
|
||||
class HTTP2Response(BaseHTTPResponse):
|
||||
# TODO: This is a woefully incomplete response object, but works for non-streaming.
|
||||
def __init__(
|
||||
self,
|
||||
status: int,
|
||||
headers: HTTPHeaderDict,
|
||||
request_url: str,
|
||||
data: bytes,
|
||||
decode_content: bool = False, # TODO: support decoding
|
||||
) -> None:
|
||||
super().__init__(
|
||||
status=status,
|
||||
headers=headers,
|
||||
# Following CPython, we map HTTP versions to major * 10 + minor integers
|
||||
version=20,
|
||||
version_string="HTTP/2",
|
||||
# No reason phrase in HTTP/2
|
||||
reason=None,
|
||||
decode_content=decode_content,
|
||||
request_url=request_url,
|
||||
)
|
||||
self._data = data
|
||||
self.length_remaining = 0
|
||||
|
||||
@property
|
||||
def data(self) -> bytes:
|
||||
return self._data
|
||||
|
||||
def get_redirect_location(self) -> None:
|
||||
return None
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
@@ -1,87 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
|
||||
class _HTTP2ProbeCache:
|
||||
__slots__ = (
|
||||
"_lock",
|
||||
"_cache_locks",
|
||||
"_cache_values",
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._cache_locks: dict[tuple[str, int], threading.RLock] = {}
|
||||
self._cache_values: dict[tuple[str, int], bool | None] = {}
|
||||
|
||||
def acquire_and_get(self, host: str, port: int) -> bool | None:
|
||||
# By the end of this block we know that
|
||||
# _cache_[values,locks] is available.
|
||||
value = None
|
||||
with self._lock:
|
||||
key = (host, port)
|
||||
try:
|
||||
value = self._cache_values[key]
|
||||
# If it's a known value we return right away.
|
||||
if value is not None:
|
||||
return value
|
||||
except KeyError:
|
||||
self._cache_locks[key] = threading.RLock()
|
||||
self._cache_values[key] = None
|
||||
|
||||
# If the value is unknown, we acquire the lock to signal
|
||||
# to the requesting thread that the probe is in progress
|
||||
# or that the current thread needs to return their findings.
|
||||
key_lock = self._cache_locks[key]
|
||||
key_lock.acquire()
|
||||
try:
|
||||
# If the by the time we get the lock the value has been
|
||||
# updated we want to return the updated value.
|
||||
value = self._cache_values[key]
|
||||
|
||||
# In case an exception like KeyboardInterrupt is raised here.
|
||||
except BaseException as e: # Defensive:
|
||||
assert not isinstance(e, KeyError) # KeyError shouldn't be possible.
|
||||
key_lock.release()
|
||||
raise
|
||||
|
||||
return value
|
||||
|
||||
def set_and_release(
|
||||
self, host: str, port: int, supports_http2: bool | None
|
||||
) -> None:
|
||||
key = (host, port)
|
||||
key_lock = self._cache_locks[key]
|
||||
with key_lock: # Uses an RLock, so can be locked again from same thread.
|
||||
if supports_http2 is None and self._cache_values[key] is not None:
|
||||
raise ValueError(
|
||||
"Cannot reset HTTP/2 support for origin after value has been set."
|
||||
) # Defensive: not expected in normal usage
|
||||
|
||||
self._cache_values[key] = supports_http2
|
||||
key_lock.release()
|
||||
|
||||
def _values(self) -> dict[tuple[str, int], bool | None]:
|
||||
"""This function is for testing purposes only. Gets the current state of the probe cache"""
|
||||
with self._lock:
|
||||
return {k: v for k, v in self._cache_values.items()}
|
||||
|
||||
def _reset(self) -> None:
|
||||
"""This function is for testing purposes only. Reset the cache values"""
|
||||
with self._lock:
|
||||
self._cache_locks = {}
|
||||
self._cache_values = {}
|
||||
|
||||
|
||||
_HTTP2_PROBE_CACHE = _HTTP2ProbeCache()
|
||||
|
||||
set_and_release = _HTTP2_PROBE_CACHE.set_and_release
|
||||
acquire_and_get = _HTTP2_PROBE_CACHE.acquire_and_get
|
||||
_values = _HTTP2_PROBE_CACHE._values
|
||||
_reset = _HTTP2_PROBE_CACHE._reset
|
||||
|
||||
__all__ = [
|
||||
"set_and_release",
|
||||
"acquire_and_get",
|
||||
]
|
||||
@@ -1,33 +1,24 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import collections
|
||||
import functools
|
||||
import logging
|
||||
import typing
|
||||
import warnings
|
||||
from types import TracebackType
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from ._collections import HTTPHeaderDict, RecentlyUsedContainer
|
||||
from ._request_methods import RequestMethods
|
||||
from .connection import ProxyConfig
|
||||
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
|
||||
from .exceptions import (
|
||||
LocationValueError,
|
||||
MaxRetryError,
|
||||
ProxySchemeUnknown,
|
||||
ProxySchemeUnsupported,
|
||||
URLSchemeUnknown,
|
||||
)
|
||||
from .response import BaseHTTPResponse
|
||||
from .util.connection import _TYPE_SOCKET_OPTIONS
|
||||
from .packages import six
|
||||
from .packages.six.moves.urllib.parse import urljoin
|
||||
from .request import RequestMethods
|
||||
from .util.proxy import connection_requires_http_tunnel
|
||||
from .util.retry import Retry
|
||||
from .util.timeout import Timeout
|
||||
from .util.url import Url, parse_url
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
import ssl
|
||||
|
||||
from typing_extensions import Self
|
||||
from .util.url import parse_url
|
||||
|
||||
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
|
||||
|
||||
@@ -39,62 +30,53 @@ SSL_KEYWORDS = (
|
||||
"cert_file",
|
||||
"cert_reqs",
|
||||
"ca_certs",
|
||||
"ca_cert_data",
|
||||
"ssl_version",
|
||||
"ssl_minimum_version",
|
||||
"ssl_maximum_version",
|
||||
"ca_cert_dir",
|
||||
"ssl_context",
|
||||
"key_password",
|
||||
"server_hostname",
|
||||
)
|
||||
# Default value for `blocksize` - a new parameter introduced to
|
||||
# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
|
||||
_DEFAULT_BLOCKSIZE = 16384
|
||||
|
||||
# All known keyword arguments that could be provided to the pool manager, its
|
||||
# pools, or the underlying connections. This is used to construct a pool key.
|
||||
_key_fields = (
|
||||
"key_scheme", # str
|
||||
"key_host", # str
|
||||
"key_port", # int
|
||||
"key_timeout", # int or float or Timeout
|
||||
"key_retries", # int or Retry
|
||||
"key_strict", # bool
|
||||
"key_block", # bool
|
||||
"key_source_address", # str
|
||||
"key_key_file", # str
|
||||
"key_key_password", # str
|
||||
"key_cert_file", # str
|
||||
"key_cert_reqs", # str
|
||||
"key_ca_certs", # str
|
||||
"key_ssl_version", # str
|
||||
"key_ca_cert_dir", # str
|
||||
"key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
|
||||
"key_maxsize", # int
|
||||
"key_headers", # dict
|
||||
"key__proxy", # parsed proxy url
|
||||
"key__proxy_headers", # dict
|
||||
"key__proxy_config", # class
|
||||
"key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples
|
||||
"key__socks_options", # dict
|
||||
"key_assert_hostname", # bool or string
|
||||
"key_assert_fingerprint", # str
|
||||
"key_server_hostname", # str
|
||||
)
|
||||
|
||||
#: The namedtuple class used to construct keys for the connection pool.
|
||||
#: All custom key schemes should include the fields in this key at a minimum.
|
||||
PoolKey = collections.namedtuple("PoolKey", _key_fields)
|
||||
|
||||
_proxy_config_fields = ("ssl_context", "use_forwarding_for_https")
|
||||
ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields)
|
||||
|
||||
|
||||
class PoolKey(typing.NamedTuple):
|
||||
"""
|
||||
All known keyword arguments that could be provided to the pool manager, its
|
||||
pools, or the underlying connections.
|
||||
|
||||
All custom key schemes should include the fields in this key at a minimum.
|
||||
"""
|
||||
|
||||
key_scheme: str
|
||||
key_host: str
|
||||
key_port: int | None
|
||||
key_timeout: Timeout | float | int | None
|
||||
key_retries: Retry | bool | int | None
|
||||
key_block: bool | None
|
||||
key_source_address: tuple[str, int] | None
|
||||
key_key_file: str | None
|
||||
key_key_password: str | None
|
||||
key_cert_file: str | None
|
||||
key_cert_reqs: str | None
|
||||
key_ca_certs: str | None
|
||||
key_ca_cert_data: str | bytes | None
|
||||
key_ssl_version: int | str | None
|
||||
key_ssl_minimum_version: ssl.TLSVersion | None
|
||||
key_ssl_maximum_version: ssl.TLSVersion | None
|
||||
key_ca_cert_dir: str | None
|
||||
key_ssl_context: ssl.SSLContext | None
|
||||
key_maxsize: int | None
|
||||
key_headers: frozenset[tuple[str, str]] | None
|
||||
key__proxy: Url | None
|
||||
key__proxy_headers: frozenset[tuple[str, str]] | None
|
||||
key__proxy_config: ProxyConfig | None
|
||||
key_socket_options: _TYPE_SOCKET_OPTIONS | None
|
||||
key__socks_options: frozenset[tuple[str, str]] | None
|
||||
key_assert_hostname: bool | str | None
|
||||
key_assert_fingerprint: str | None
|
||||
key_server_hostname: str | None
|
||||
key_blocksize: int | None
|
||||
|
||||
|
||||
def _default_key_normalizer(
|
||||
key_class: type[PoolKey], request_context: dict[str, typing.Any]
|
||||
) -> PoolKey:
|
||||
def _default_key_normalizer(key_class, request_context):
|
||||
"""
|
||||
Create a pool key out of a request context dictionary.
|
||||
|
||||
@@ -140,10 +122,6 @@ def _default_key_normalizer(
|
||||
if field not in context:
|
||||
context[field] = None
|
||||
|
||||
# Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
|
||||
if context.get("key_blocksize") is None:
|
||||
context["key_blocksize"] = _DEFAULT_BLOCKSIZE
|
||||
|
||||
return key_class(**context)
|
||||
|
||||
|
||||
@@ -176,50 +154,23 @@ class PoolManager(RequestMethods):
|
||||
Additional parameters are used to create fresh
|
||||
:class:`urllib3.connectionpool.ConnectionPool` instances.
|
||||
|
||||
Example:
|
||||
Example::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
http = urllib3.PoolManager(num_pools=2)
|
||||
|
||||
resp1 = http.request("GET", "https://google.com/")
|
||||
resp2 = http.request("GET", "https://google.com/mail")
|
||||
resp3 = http.request("GET", "https://yahoo.com/")
|
||||
|
||||
print(len(http.pools))
|
||||
# 2
|
||||
>>> manager = PoolManager(num_pools=2)
|
||||
>>> r = manager.request('GET', 'http://google.com/')
|
||||
>>> r = manager.request('GET', 'http://google.com/mail')
|
||||
>>> r = manager.request('GET', 'http://yahoo.com/')
|
||||
>>> len(manager.pools)
|
||||
2
|
||||
|
||||
"""
|
||||
|
||||
proxy: Url | None = None
|
||||
proxy_config: ProxyConfig | None = None
|
||||
proxy = None
|
||||
proxy_config = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_pools: int = 10,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
**connection_pool_kw: typing.Any,
|
||||
) -> None:
|
||||
super().__init__(headers)
|
||||
# PoolManager handles redirects itself in PoolManager.urlopen().
|
||||
# It always passes redirect=False to the underlying connection pool to
|
||||
# suppress per-pool redirect handling. If the user supplied a non-Retry
|
||||
# value (int/bool/etc) for retries and we let the pool normalize it
|
||||
# while redirect=False, the resulting Retry object would have redirect
|
||||
# handling disabled, which can interfere with PoolManager's own
|
||||
# redirect logic. Normalize here so redirects remain governed solely by
|
||||
# PoolManager logic.
|
||||
if "retries" in connection_pool_kw:
|
||||
retries = connection_pool_kw["retries"]
|
||||
if not isinstance(retries, Retry):
|
||||
retries = Retry.from_int(retries)
|
||||
connection_pool_kw = connection_pool_kw.copy()
|
||||
connection_pool_kw["retries"] = retries
|
||||
def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
|
||||
RequestMethods.__init__(self, headers)
|
||||
self.connection_pool_kw = connection_pool_kw
|
||||
|
||||
self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
|
||||
self.pools = RecentlyUsedContainer(num_pools)
|
||||
|
||||
# Locally set the pool classes and keys so other PoolManagers can
|
||||
@@ -227,26 +178,15 @@ class PoolManager(RequestMethods):
|
||||
self.pool_classes_by_scheme = pool_classes_by_scheme
|
||||
self.key_fn_by_scheme = key_fn_by_scheme.copy()
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_val: BaseException | None,
|
||||
exc_tb: TracebackType | None,
|
||||
) -> typing.Literal[False]:
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.clear()
|
||||
# Return False to re-raise any potential exceptions
|
||||
return False
|
||||
|
||||
def _new_pool(
|
||||
self,
|
||||
scheme: str,
|
||||
host: str,
|
||||
port: int,
|
||||
request_context: dict[str, typing.Any] | None = None,
|
||||
) -> HTTPConnectionPool:
|
||||
def _new_pool(self, scheme, host, port, request_context=None):
|
||||
"""
|
||||
Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
|
||||
any additional pool keyword arguments.
|
||||
@@ -256,15 +196,10 @@ class PoolManager(RequestMethods):
|
||||
connection pools handed out by :meth:`connection_from_url` and
|
||||
companion methods. It is intended to be overridden for customization.
|
||||
"""
|
||||
pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
|
||||
pool_cls = self.pool_classes_by_scheme[scheme]
|
||||
if request_context is None:
|
||||
request_context = self.connection_pool_kw.copy()
|
||||
|
||||
# Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
|
||||
# set to 'None' in the request_context.
|
||||
if request_context.get("blocksize") is None:
|
||||
request_context["blocksize"] = _DEFAULT_BLOCKSIZE
|
||||
|
||||
# Although the context has everything necessary to create the pool,
|
||||
# this function has historically only used the scheme, host, and port
|
||||
# in the positional args. When an API change is acceptable these can
|
||||
@@ -278,7 +213,7 @@ class PoolManager(RequestMethods):
|
||||
|
||||
return pool_cls(host, port, **request_context)
|
||||
|
||||
def clear(self) -> None:
|
||||
def clear(self):
|
||||
"""
|
||||
Empty our store of pools and direct them all to close.
|
||||
|
||||
@@ -287,13 +222,7 @@ class PoolManager(RequestMethods):
|
||||
"""
|
||||
self.pools.clear()
|
||||
|
||||
def connection_from_host(
|
||||
self,
|
||||
host: str | None,
|
||||
port: int | None = None,
|
||||
scheme: str | None = "http",
|
||||
pool_kwargs: dict[str, typing.Any] | None = None,
|
||||
) -> HTTPConnectionPool:
|
||||
def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
|
||||
"""
|
||||
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
|
||||
|
||||
@@ -316,23 +245,13 @@ class PoolManager(RequestMethods):
|
||||
|
||||
return self.connection_from_context(request_context)
|
||||
|
||||
def connection_from_context(
|
||||
self, request_context: dict[str, typing.Any]
|
||||
) -> HTTPConnectionPool:
|
||||
def connection_from_context(self, request_context):
|
||||
"""
|
||||
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
|
||||
|
||||
``request_context`` must at least contain the ``scheme`` key and its
|
||||
value must be a key in ``key_fn_by_scheme`` instance variable.
|
||||
"""
|
||||
if "strict" in request_context:
|
||||
warnings.warn(
|
||||
"The 'strict' parameter is no longer needed on Python 3+. "
|
||||
"This will raise an error in urllib3 v3.0.",
|
||||
FutureWarning,
|
||||
)
|
||||
request_context.pop("strict")
|
||||
|
||||
scheme = request_context["scheme"].lower()
|
||||
pool_key_constructor = self.key_fn_by_scheme.get(scheme)
|
||||
if not pool_key_constructor:
|
||||
@@ -341,9 +260,7 @@ class PoolManager(RequestMethods):
|
||||
|
||||
return self.connection_from_pool_key(pool_key, request_context=request_context)
|
||||
|
||||
def connection_from_pool_key(
|
||||
self, pool_key: PoolKey, request_context: dict[str, typing.Any]
|
||||
) -> HTTPConnectionPool:
|
||||
def connection_from_pool_key(self, pool_key, request_context=None):
|
||||
"""
|
||||
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
|
||||
|
||||
@@ -367,9 +284,7 @@ class PoolManager(RequestMethods):
|
||||
|
||||
return pool
|
||||
|
||||
def connection_from_url(
|
||||
self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
|
||||
) -> HTTPConnectionPool:
|
||||
def connection_from_url(self, url, pool_kwargs=None):
|
||||
"""
|
||||
Similar to :func:`urllib3.connectionpool.connection_from_url`.
|
||||
|
||||
@@ -385,9 +300,7 @@ class PoolManager(RequestMethods):
|
||||
u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
|
||||
)
|
||||
|
||||
def _merge_pool_kwargs(
|
||||
self, override: dict[str, typing.Any] | None
|
||||
) -> dict[str, typing.Any]:
|
||||
def _merge_pool_kwargs(self, override):
|
||||
"""
|
||||
Merge a dictionary of override values for self.connection_pool_kw.
|
||||
|
||||
@@ -407,7 +320,7 @@ class PoolManager(RequestMethods):
|
||||
base_pool_kwargs[key] = value
|
||||
return base_pool_kwargs
|
||||
|
||||
def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
|
||||
def _proxy_requires_url_absolute_form(self, parsed_url):
|
||||
"""
|
||||
Indicates if the proxy requires the complete destination URL in the
|
||||
request. Normally this is only needed when not using an HTTP CONNECT
|
||||
@@ -420,9 +333,24 @@ class PoolManager(RequestMethods):
|
||||
self.proxy, self.proxy_config, parsed_url.scheme
|
||||
)
|
||||
|
||||
def urlopen( # type: ignore[override]
|
||||
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
|
||||
) -> BaseHTTPResponse:
|
||||
def _validate_proxy_scheme_url_selection(self, url_scheme):
|
||||
"""
|
||||
Validates that were not attempting to do TLS in TLS connections on
|
||||
Python2 or with unsupported SSL implementations.
|
||||
"""
|
||||
if self.proxy is None or url_scheme != "https":
|
||||
return
|
||||
|
||||
if self.proxy.scheme != "https":
|
||||
return
|
||||
|
||||
if six.PY2 and not self.proxy_config.use_forwarding_for_https:
|
||||
raise ProxySchemeUnsupported(
|
||||
"Contacting HTTPS destinations through HTTPS proxies "
|
||||
"'via CONNECT tunnels' is not supported in Python 2"
|
||||
)
|
||||
|
||||
def urlopen(self, method, url, redirect=True, **kw):
|
||||
"""
|
||||
Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
|
||||
with custom cross-host redirect logic and only sends the request-uri
|
||||
@@ -432,16 +360,7 @@ class PoolManager(RequestMethods):
|
||||
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
|
||||
"""
|
||||
u = parse_url(url)
|
||||
|
||||
if u.scheme is None:
|
||||
warnings.warn(
|
||||
"URLs without a scheme (ie 'https://') are deprecated and will raise an error "
|
||||
"in urllib3 v3.0. To avoid this FutureWarning ensure all URLs "
|
||||
"start with 'https://' or 'http://'. Read more in this issue: "
|
||||
"https://github.com/urllib3/urllib3/issues/2920",
|
||||
category=FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
self._validate_proxy_scheme_url_selection(u.scheme)
|
||||
|
||||
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
|
||||
|
||||
@@ -449,7 +368,7 @@ class PoolManager(RequestMethods):
|
||||
kw["redirect"] = False
|
||||
|
||||
if "headers" not in kw:
|
||||
kw["headers"] = self.headers
|
||||
kw["headers"] = self.headers.copy()
|
||||
|
||||
if self._proxy_requires_url_absolute_form(u):
|
||||
response = conn.urlopen(method, url, **kw)
|
||||
@@ -470,7 +389,7 @@ class PoolManager(RequestMethods):
|
||||
kw["body"] = None
|
||||
kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
|
||||
|
||||
retries = kw.get("retries", response.retries)
|
||||
retries = kw.get("retries")
|
||||
if not isinstance(retries, Retry):
|
||||
retries = Retry.from_int(retries, redirect=redirect)
|
||||
|
||||
@@ -480,11 +399,10 @@ class PoolManager(RequestMethods):
|
||||
if retries.remove_headers_on_redirect and not conn.is_same_host(
|
||||
redirect_location
|
||||
):
|
||||
new_headers = kw["headers"].copy()
|
||||
for header in kw["headers"]:
|
||||
headers = list(six.iterkeys(kw["headers"]))
|
||||
for header in headers:
|
||||
if header.lower() in retries.remove_headers_on_redirect:
|
||||
new_headers.pop(header, None)
|
||||
kw["headers"] = new_headers
|
||||
kw["headers"].pop(header, None)
|
||||
|
||||
try:
|
||||
retries = retries.increment(method, url, response=response, _pool=conn)
|
||||
@@ -530,53 +448,37 @@ class ProxyManager(PoolManager):
|
||||
private. IP address, target hostname, SNI, and port are always visible
|
||||
to an HTTPS proxy even when this flag is disabled.
|
||||
|
||||
:param proxy_assert_hostname:
|
||||
The hostname of the certificate to verify against.
|
||||
|
||||
:param proxy_assert_fingerprint:
|
||||
The fingerprint of the certificate to verify against.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
proxy = urllib3.ProxyManager("https://localhost:3128/")
|
||||
|
||||
resp1 = proxy.request("GET", "http://google.com/")
|
||||
resp2 = proxy.request("GET", "http://httpbin.org/")
|
||||
|
||||
# One pool was shared by both plain HTTP requests.
|
||||
print(len(proxy.pools))
|
||||
# 1
|
||||
|
||||
resp3 = proxy.request("GET", "https://httpbin.org/")
|
||||
resp4 = proxy.request("GET", "https://twitter.com/")
|
||||
|
||||
# A separate pool was added for each HTTPS target.
|
||||
print(len(proxy.pools))
|
||||
# 3
|
||||
>>> proxy = urllib3.ProxyManager('http://localhost:3128/')
|
||||
>>> r1 = proxy.request('GET', 'http://google.com/')
|
||||
>>> r2 = proxy.request('GET', 'http://httpbin.org/')
|
||||
>>> len(proxy.pools)
|
||||
1
|
||||
>>> r3 = proxy.request('GET', 'https://httpbin.org/')
|
||||
>>> r4 = proxy.request('GET', 'https://twitter.com/')
|
||||
>>> len(proxy.pools)
|
||||
3
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
proxy_url: str,
|
||||
num_pools: int = 10,
|
||||
headers: typing.Mapping[str, str] | None = None,
|
||||
proxy_headers: typing.Mapping[str, str] | None = None,
|
||||
proxy_ssl_context: ssl.SSLContext | None = None,
|
||||
use_forwarding_for_https: bool = False,
|
||||
proxy_assert_hostname: None | str | typing.Literal[False] = None,
|
||||
proxy_assert_fingerprint: str | None = None,
|
||||
**connection_pool_kw: typing.Any,
|
||||
) -> None:
|
||||
proxy_url,
|
||||
num_pools=10,
|
||||
headers=None,
|
||||
proxy_headers=None,
|
||||
proxy_ssl_context=None,
|
||||
use_forwarding_for_https=False,
|
||||
**connection_pool_kw
|
||||
):
|
||||
|
||||
if isinstance(proxy_url, HTTPConnectionPool):
|
||||
str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
|
||||
else:
|
||||
str_proxy_url = proxy_url
|
||||
proxy = parse_url(str_proxy_url)
|
||||
proxy_url = "%s://%s:%i" % (
|
||||
proxy_url.scheme,
|
||||
proxy_url.host,
|
||||
proxy_url.port,
|
||||
)
|
||||
proxy = parse_url(proxy_url)
|
||||
|
||||
if proxy.scheme not in ("http", "https"):
|
||||
raise ProxySchemeUnknown(proxy.scheme)
|
||||
@@ -588,38 +490,25 @@ class ProxyManager(PoolManager):
|
||||
self.proxy = proxy
|
||||
self.proxy_headers = proxy_headers or {}
|
||||
self.proxy_ssl_context = proxy_ssl_context
|
||||
self.proxy_config = ProxyConfig(
|
||||
proxy_ssl_context,
|
||||
use_forwarding_for_https,
|
||||
proxy_assert_hostname,
|
||||
proxy_assert_fingerprint,
|
||||
)
|
||||
self.proxy_config = ProxyConfig(proxy_ssl_context, use_forwarding_for_https)
|
||||
|
||||
connection_pool_kw["_proxy"] = self.proxy
|
||||
connection_pool_kw["_proxy_headers"] = self.proxy_headers
|
||||
connection_pool_kw["_proxy_config"] = self.proxy_config
|
||||
|
||||
super().__init__(num_pools, headers, **connection_pool_kw)
|
||||
super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
|
||||
|
||||
def connection_from_host(
|
||||
self,
|
||||
host: str | None,
|
||||
port: int | None = None,
|
||||
scheme: str | None = "http",
|
||||
pool_kwargs: dict[str, typing.Any] | None = None,
|
||||
) -> HTTPConnectionPool:
|
||||
def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
|
||||
if scheme == "https":
|
||||
return super().connection_from_host(
|
||||
return super(ProxyManager, self).connection_from_host(
|
||||
host, port, scheme, pool_kwargs=pool_kwargs
|
||||
)
|
||||
|
||||
return super().connection_from_host(
|
||||
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
|
||||
return super(ProxyManager, self).connection_from_host(
|
||||
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs
|
||||
)
|
||||
|
||||
def _set_proxy_headers(
|
||||
self, url: str, headers: typing.Mapping[str, str] | None = None
|
||||
) -> typing.Mapping[str, str]:
|
||||
def _set_proxy_headers(self, url, headers=None):
|
||||
"""
|
||||
Sets headers needed by proxies: specifically, the Accept and Host
|
||||
headers. Only sets headers not provided by the user.
|
||||
@@ -634,9 +523,7 @@ class ProxyManager(PoolManager):
|
||||
headers_.update(headers)
|
||||
return headers_
|
||||
|
||||
def urlopen( # type: ignore[override]
|
||||
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
|
||||
) -> BaseHTTPResponse:
|
||||
def urlopen(self, method, url, redirect=True, **kw):
|
||||
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
|
||||
u = parse_url(url)
|
||||
if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
|
||||
@@ -646,8 +533,8 @@ class ProxyManager(PoolManager):
|
||||
headers = kw.get("headers", self.headers)
|
||||
kw["headers"] = self._set_proxy_headers(url, headers)
|
||||
|
||||
return super().urlopen(method, url, redirect=redirect, **kw)
|
||||
return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
|
||||
|
||||
|
||||
def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
|
||||
def proxy_from_url(url, **kw):
|
||||
return ProxyManager(proxy_url=url, **kw)
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# Instruct type checkers to look for inline type annotations in this package.
|
||||
# See PEP 561.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,39 +1,46 @@
|
||||
# For backwards compatibility, provide imports that used to be here.
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
# For backwards compatibility, provide imports that used to be here.
|
||||
from .connection import is_connection_dropped
|
||||
from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers
|
||||
from .response import is_fp_closed
|
||||
from .retry import Retry
|
||||
from .ssl_ import (
|
||||
ALPN_PROTOCOLS,
|
||||
HAS_SNI,
|
||||
IS_PYOPENSSL,
|
||||
IS_SECURETRANSPORT,
|
||||
PROTOCOL_TLS,
|
||||
SSLContext,
|
||||
assert_fingerprint,
|
||||
create_urllib3_context,
|
||||
resolve_cert_reqs,
|
||||
resolve_ssl_version,
|
||||
ssl_wrap_socket,
|
||||
)
|
||||
from .timeout import Timeout
|
||||
from .url import Url, parse_url
|
||||
from .timeout import Timeout, current_time
|
||||
from .url import Url, get_host, parse_url, split_first
|
||||
from .wait import wait_for_read, wait_for_write
|
||||
|
||||
__all__ = (
|
||||
"HAS_SNI",
|
||||
"IS_PYOPENSSL",
|
||||
"IS_SECURETRANSPORT",
|
||||
"SSLContext",
|
||||
"PROTOCOL_TLS",
|
||||
"ALPN_PROTOCOLS",
|
||||
"Retry",
|
||||
"Timeout",
|
||||
"Url",
|
||||
"assert_fingerprint",
|
||||
"create_urllib3_context",
|
||||
"current_time",
|
||||
"is_connection_dropped",
|
||||
"is_fp_closed",
|
||||
"get_host",
|
||||
"parse_url",
|
||||
"make_headers",
|
||||
"resolve_cert_reqs",
|
||||
"resolve_ssl_version",
|
||||
"split_first",
|
||||
"ssl_wrap_socket",
|
||||
"wait_for_read",
|
||||
"wait_for_write",
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,23 +1,33 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import socket
|
||||
import typing
|
||||
|
||||
from ..contrib import _appengine_environ
|
||||
from ..exceptions import LocationParseError
|
||||
from .timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
|
||||
|
||||
_TYPE_SOCKET_OPTIONS = list[tuple[int, int, typing.Union[int, bytes]]]
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from .._base_connection import BaseHTTPConnection
|
||||
from ..packages import six
|
||||
from .wait import NoWayToWaitForSocketError, wait_for_read
|
||||
|
||||
|
||||
def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specific
|
||||
def is_connection_dropped(conn): # Platform-specific
|
||||
"""
|
||||
Returns True if the connection is dropped and should be closed.
|
||||
:param conn: :class:`urllib3.connection.HTTPConnection` object.
|
||||
|
||||
:param conn:
|
||||
:class:`http.client.HTTPConnection` object.
|
||||
|
||||
Note: For platforms like AppEngine, this will always return ``False`` to
|
||||
let the platform handle connection recycling transparently for us.
|
||||
"""
|
||||
return not conn.is_connected
|
||||
sock = getattr(conn, "sock", False)
|
||||
if sock is False: # Platform-specific: AppEngine
|
||||
return False
|
||||
if sock is None: # Connection already closed (such as by httplib).
|
||||
return True
|
||||
try:
|
||||
# Returns True if readable, which here means it's been dropped
|
||||
return wait_for_read(sock, timeout=0.0)
|
||||
except NoWayToWaitForSocketError: # Platform-specific: AppEngine
|
||||
return False
|
||||
|
||||
|
||||
# This function is copied from socket.py in the Python 2.7 standard
|
||||
@@ -25,11 +35,11 @@ def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specifi
|
||||
# One additional modification is that we avoid binding to IPv6 servers
|
||||
# discovered in DNS if the system doesn't have IPv6 functionality.
|
||||
def create_connection(
|
||||
address: tuple[str, int],
|
||||
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
|
||||
source_address: tuple[str, int] | None = None,
|
||||
socket_options: _TYPE_SOCKET_OPTIONS | None = None,
|
||||
) -> socket.socket:
|
||||
address,
|
||||
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
|
||||
source_address=None,
|
||||
socket_options=None,
|
||||
):
|
||||
"""Connect to *address* and return the socket object.
|
||||
|
||||
Convenience function. Connect to *address* (a 2-tuple ``(host,
|
||||
@@ -55,7 +65,9 @@ def create_connection(
|
||||
try:
|
||||
host.encode("idna")
|
||||
except UnicodeError:
|
||||
raise LocationParseError(f"'{host}', label empty or too long") from None
|
||||
return six.raise_from(
|
||||
LocationParseError(u"'%s', label empty or too long" % host), None
|
||||
)
|
||||
|
||||
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
|
||||
af, socktype, proto, canonname, sa = res
|
||||
@@ -66,33 +78,26 @@ def create_connection(
|
||||
# If provided, set socket level options before connecting.
|
||||
_set_socket_options(sock, socket_options)
|
||||
|
||||
if timeout is not _DEFAULT_TIMEOUT:
|
||||
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
|
||||
sock.settimeout(timeout)
|
||||
if source_address:
|
||||
sock.bind(source_address)
|
||||
sock.connect(sa)
|
||||
# Break explicitly a reference cycle
|
||||
err = None
|
||||
return sock
|
||||
|
||||
except OSError as _:
|
||||
err = _
|
||||
except socket.error as e:
|
||||
err = e
|
||||
if sock is not None:
|
||||
sock.close()
|
||||
sock = None
|
||||
|
||||
if err is not None:
|
||||
try:
|
||||
raise err
|
||||
finally:
|
||||
# Break explicitly a reference cycle
|
||||
err = None
|
||||
else:
|
||||
raise OSError("getaddrinfo returns an empty list")
|
||||
raise err
|
||||
|
||||
raise socket.error("getaddrinfo returns an empty list")
|
||||
|
||||
|
||||
def _set_socket_options(
|
||||
sock: socket.socket, options: _TYPE_SOCKET_OPTIONS | None
|
||||
) -> None:
|
||||
def _set_socket_options(sock, options):
|
||||
if options is None:
|
||||
return
|
||||
|
||||
@@ -100,7 +105,7 @@ def _set_socket_options(
|
||||
sock.setsockopt(*opt)
|
||||
|
||||
|
||||
def allowed_gai_family() -> socket.AddressFamily:
|
||||
def allowed_gai_family():
|
||||
"""This function is designed to work in the context of
|
||||
getaddrinfo, where family=socket.AF_UNSPEC is the default and
|
||||
will perform a DNS search for both IPv6 and IPv4 records."""
|
||||
@@ -111,11 +116,18 @@ def allowed_gai_family() -> socket.AddressFamily:
|
||||
return family
|
||||
|
||||
|
||||
def _has_ipv6(host: str) -> bool:
|
||||
def _has_ipv6(host):
|
||||
"""Returns True if the system can bind an IPv6 address."""
|
||||
sock = None
|
||||
has_ipv6 = False
|
||||
|
||||
# App Engine doesn't support IPV6 sockets and actually has a quota on the
|
||||
# number of sockets that can be used, so just early out here instead of
|
||||
# creating a socket needlessly.
|
||||
# See https://github.com/urllib3/urllib3/issues/1446
|
||||
if _appengine_environ.is_appengine_sandbox():
|
||||
return False
|
||||
|
||||
if socket.has_ipv6:
|
||||
# has_ipv6 returns true if cPython was compiled with IPv6 support.
|
||||
# It does not tell us if the system has IPv6 support enabled. To
|
||||
|
||||
@@ -1,18 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from .url import Url
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ..connection import ProxyConfig
|
||||
from .ssl_ import create_urllib3_context, resolve_cert_reqs, resolve_ssl_version
|
||||
|
||||
|
||||
def connection_requires_http_tunnel(
|
||||
proxy_url: Url | None = None,
|
||||
proxy_config: ProxyConfig | None = None,
|
||||
destination_scheme: str | None = None,
|
||||
) -> bool:
|
||||
proxy_url=None, proxy_config=None, destination_scheme=None
|
||||
):
|
||||
"""
|
||||
Returns True if the connection requires an HTTP CONNECT through the proxy.
|
||||
|
||||
@@ -41,3 +32,26 @@ def connection_requires_http_tunnel(
|
||||
|
||||
# Otherwise always use a tunnel.
|
||||
return True
|
||||
|
||||
|
||||
def create_proxy_ssl_context(
|
||||
ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None
|
||||
):
|
||||
"""
|
||||
Generates a default proxy ssl context if one hasn't been provided by the
|
||||
user.
|
||||
"""
|
||||
ssl_context = create_urllib3_context(
|
||||
ssl_version=resolve_ssl_version(ssl_version),
|
||||
cert_reqs=resolve_cert_reqs(cert_reqs),
|
||||
)
|
||||
|
||||
if (
|
||||
not ca_certs
|
||||
and not ca_cert_dir
|
||||
and not ca_cert_data
|
||||
and hasattr(ssl_context, "load_default_certs")
|
||||
):
|
||||
ssl_context.load_default_certs()
|
||||
|
||||
return ssl_context
|
||||
|
||||
@@ -1,16 +1,9 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import io
|
||||
import sys
|
||||
import typing
|
||||
from base64 import b64encode
|
||||
from enum import Enum
|
||||
|
||||
from ..exceptions import UnrewindableBodyError
|
||||
from .util import to_bytes
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from typing import Final
|
||||
from ..packages.six import b, integer_types
|
||||
|
||||
# Pass as a value within ``headers`` to skip
|
||||
# emitting some HTTP headers that are added automatically.
|
||||
@@ -22,49 +15,25 @@ SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"])
|
||||
ACCEPT_ENCODING = "gzip,deflate"
|
||||
try:
|
||||
try:
|
||||
import brotlicffi as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401
|
||||
import brotlicffi as _unused_module_brotli # noqa: F401
|
||||
except ImportError:
|
||||
import brotli as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401
|
||||
import brotli as _unused_module_brotli # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
ACCEPT_ENCODING += ",br"
|
||||
|
||||
try:
|
||||
if sys.version_info >= (3, 14):
|
||||
from compression import zstd as _unused_module_zstd # noqa: F401
|
||||
else:
|
||||
from backports import zstd as _unused_module_zstd # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
ACCEPT_ENCODING += ",zstd"
|
||||
|
||||
|
||||
class _TYPE_FAILEDTELL(Enum):
|
||||
token = 0
|
||||
|
||||
|
||||
_FAILEDTELL: Final[_TYPE_FAILEDTELL] = _TYPE_FAILEDTELL.token
|
||||
|
||||
_TYPE_BODY_POSITION = typing.Union[int, _TYPE_FAILEDTELL]
|
||||
|
||||
# When sending a request with these methods we aren't expecting
|
||||
# a body so don't need to set an explicit 'Content-Length: 0'
|
||||
# The reason we do this in the negative instead of tracking methods
|
||||
# which 'should' have a body is because unknown methods should be
|
||||
# treated as if they were 'POST' which *does* expect a body.
|
||||
_METHODS_NOT_EXPECTING_BODY = {"GET", "HEAD", "DELETE", "TRACE", "OPTIONS", "CONNECT"}
|
||||
_FAILEDTELL = object()
|
||||
|
||||
|
||||
def make_headers(
|
||||
keep_alive: bool | None = None,
|
||||
accept_encoding: bool | list[str] | str | None = None,
|
||||
user_agent: str | None = None,
|
||||
basic_auth: str | None = None,
|
||||
proxy_basic_auth: str | None = None,
|
||||
disable_cache: bool | None = None,
|
||||
) -> dict[str, str]:
|
||||
keep_alive=None,
|
||||
accept_encoding=None,
|
||||
user_agent=None,
|
||||
basic_auth=None,
|
||||
proxy_basic_auth=None,
|
||||
disable_cache=None,
|
||||
):
|
||||
"""
|
||||
Shortcuts for generating request headers.
|
||||
|
||||
@@ -73,11 +42,7 @@ def make_headers(
|
||||
|
||||
:param accept_encoding:
|
||||
Can be a boolean, list, or string.
|
||||
``True`` translates to 'gzip,deflate'. If the dependencies for
|
||||
Brotli (either the ``brotli`` or ``brotlicffi`` package) and/or
|
||||
Zstandard (the ``backports.zstd`` package for Python before 3.14)
|
||||
algorithms are installed, then their encodings are
|
||||
included in the string ('br' and 'zstd', respectively).
|
||||
``True`` translates to 'gzip,deflate'.
|
||||
List will get joined by comma.
|
||||
String will be used as provided.
|
||||
|
||||
@@ -96,18 +61,14 @@ def make_headers(
|
||||
:param disable_cache:
|
||||
If ``True``, adds 'cache-control: no-cache' header.
|
||||
|
||||
Example:
|
||||
Example::
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import urllib3
|
||||
|
||||
print(urllib3.util.make_headers(keep_alive=True, user_agent="Batman/1.0"))
|
||||
# {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
|
||||
print(urllib3.util.make_headers(accept_encoding=True))
|
||||
# {'accept-encoding': 'gzip,deflate'}
|
||||
>>> make_headers(keep_alive=True, user_agent="Batman/1.0")
|
||||
{'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
|
||||
>>> make_headers(accept_encoding=True)
|
||||
{'accept-encoding': 'gzip,deflate'}
|
||||
"""
|
||||
headers: dict[str, str] = {}
|
||||
headers = {}
|
||||
if accept_encoding:
|
||||
if isinstance(accept_encoding, str):
|
||||
pass
|
||||
@@ -124,14 +85,12 @@ def make_headers(
|
||||
headers["connection"] = "keep-alive"
|
||||
|
||||
if basic_auth:
|
||||
headers["authorization"] = (
|
||||
f"Basic {b64encode(basic_auth.encode('latin-1')).decode()}"
|
||||
)
|
||||
headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8")
|
||||
|
||||
if proxy_basic_auth:
|
||||
headers["proxy-authorization"] = (
|
||||
f"Basic {b64encode(proxy_basic_auth.encode('latin-1')).decode()}"
|
||||
)
|
||||
headers["proxy-authorization"] = "Basic " + b64encode(
|
||||
b(proxy_basic_auth)
|
||||
).decode("utf-8")
|
||||
|
||||
if disable_cache:
|
||||
headers["cache-control"] = "no-cache"
|
||||
@@ -139,9 +98,7 @@ def make_headers(
|
||||
return headers
|
||||
|
||||
|
||||
def set_file_position(
|
||||
body: typing.Any, pos: _TYPE_BODY_POSITION | None
|
||||
) -> _TYPE_BODY_POSITION | None:
|
||||
def set_file_position(body, pos):
|
||||
"""
|
||||
If a position is provided, move file to that point.
|
||||
Otherwise, we'll attempt to record a position for future use.
|
||||
@@ -151,7 +108,7 @@ def set_file_position(
|
||||
elif getattr(body, "tell", None) is not None:
|
||||
try:
|
||||
pos = body.tell()
|
||||
except OSError:
|
||||
except (IOError, OSError):
|
||||
# This differentiates from None, allowing us to catch
|
||||
# a failed `tell()` later when trying to rewind the body.
|
||||
pos = _FAILEDTELL
|
||||
@@ -159,7 +116,7 @@ def set_file_position(
|
||||
return pos
|
||||
|
||||
|
||||
def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -> None:
|
||||
def rewind_body(body, body_pos):
|
||||
"""
|
||||
Attempt to rewind body to a certain position.
|
||||
Primarily used for request redirects and retries.
|
||||
@@ -171,13 +128,13 @@ def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -
|
||||
Position to seek to in file.
|
||||
"""
|
||||
body_seek = getattr(body, "seek", None)
|
||||
if body_seek is not None and isinstance(body_pos, int):
|
||||
if body_seek is not None and isinstance(body_pos, integer_types):
|
||||
try:
|
||||
body_seek(body_pos)
|
||||
except OSError as e:
|
||||
except (IOError, OSError):
|
||||
raise UnrewindableBodyError(
|
||||
"An error occurred when rewinding request body for redirect/retry."
|
||||
) from e
|
||||
)
|
||||
elif body_pos is _FAILEDTELL:
|
||||
raise UnrewindableBodyError(
|
||||
"Unable to record file position for rewinding "
|
||||
@@ -185,79 +142,5 @@ def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"body_pos must be of type integer, instead it was {type(body_pos)}."
|
||||
"body_pos must be of type integer, instead it was %s." % type(body_pos)
|
||||
)
|
||||
|
||||
|
||||
class ChunksAndContentLength(typing.NamedTuple):
|
||||
chunks: typing.Iterable[bytes] | None
|
||||
content_length: int | None
|
||||
|
||||
|
||||
def body_to_chunks(
|
||||
body: typing.Any | None, method: str, blocksize: int
|
||||
) -> ChunksAndContentLength:
|
||||
"""Takes the HTTP request method, body, and blocksize and
|
||||
transforms them into an iterable of chunks to pass to
|
||||
socket.sendall() and an optional 'Content-Length' header.
|
||||
|
||||
A 'Content-Length' of 'None' indicates the length of the body
|
||||
can't be determined so should use 'Transfer-Encoding: chunked'
|
||||
for framing instead.
|
||||
"""
|
||||
|
||||
chunks: typing.Iterable[bytes] | None
|
||||
content_length: int | None
|
||||
|
||||
# No body, we need to make a recommendation on 'Content-Length'
|
||||
# based on whether that request method is expected to have
|
||||
# a body or not.
|
||||
if body is None:
|
||||
chunks = None
|
||||
if method.upper() not in _METHODS_NOT_EXPECTING_BODY:
|
||||
content_length = 0
|
||||
else:
|
||||
content_length = None
|
||||
|
||||
# Bytes or strings become bytes
|
||||
elif isinstance(body, (str, bytes)):
|
||||
chunks = (to_bytes(body),)
|
||||
content_length = len(chunks[0])
|
||||
|
||||
# File-like object, TODO: use seek() and tell() for length?
|
||||
elif hasattr(body, "read"):
|
||||
|
||||
def chunk_readable() -> typing.Iterable[bytes]:
|
||||
encode = isinstance(body, io.TextIOBase)
|
||||
while True:
|
||||
datablock = body.read(blocksize)
|
||||
if not datablock:
|
||||
break
|
||||
if encode:
|
||||
datablock = datablock.encode("utf-8")
|
||||
yield datablock
|
||||
|
||||
chunks = chunk_readable()
|
||||
content_length = None
|
||||
|
||||
# Otherwise we need to start checking via duck-typing.
|
||||
else:
|
||||
try:
|
||||
# Check if the body implements the buffer API.
|
||||
mv = memoryview(body)
|
||||
except TypeError:
|
||||
try:
|
||||
# Check if the body is an iterable
|
||||
chunks = iter(body)
|
||||
content_length = None
|
||||
except TypeError:
|
||||
raise TypeError(
|
||||
f"'body' must be a bytes-like object, file-like "
|
||||
f"object, or iterable. Instead was {body!r}"
|
||||
) from None
|
||||
else:
|
||||
# Since it implements the buffer API can be passed directly to socket.sendall()
|
||||
chunks = (body,)
|
||||
content_length = mv.nbytes
|
||||
|
||||
return ChunksAndContentLength(chunks=chunks, content_length=content_length)
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import http.client as httplib
|
||||
from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect
|
||||
|
||||
from ..exceptions import HeaderParsingError
|
||||
from ..packages.six.moves import http_client as httplib
|
||||
|
||||
|
||||
def is_fp_closed(obj: object) -> bool:
|
||||
def is_fp_closed(obj):
|
||||
"""
|
||||
Checks whether a given file-like object is closed.
|
||||
|
||||
@@ -17,27 +17,27 @@ def is_fp_closed(obj: object) -> bool:
|
||||
try:
|
||||
# Check `isclosed()` first, in case Python3 doesn't set `closed`.
|
||||
# GH Issue #928
|
||||
return obj.isclosed() # type: ignore[no-any-return, attr-defined]
|
||||
return obj.isclosed()
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Check via the official file-like-object way.
|
||||
return obj.closed # type: ignore[no-any-return, attr-defined]
|
||||
return obj.closed
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Check if the object is a container for another file-like object that
|
||||
# gets released on exhaustion (e.g. HTTPResponse).
|
||||
return obj.fp is None # type: ignore[attr-defined]
|
||||
return obj.fp is None
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
raise ValueError("Unable to determine whether fp is closed.")
|
||||
|
||||
|
||||
def assert_header_parsing(headers: httplib.HTTPMessage) -> None:
|
||||
def assert_header_parsing(headers):
|
||||
"""
|
||||
Asserts whether all headers have been successfully parsed.
|
||||
Extracts encountered errors from the result of parsing headers.
|
||||
@@ -53,49 +53,55 @@ def assert_header_parsing(headers: httplib.HTTPMessage) -> None:
|
||||
# This will fail silently if we pass in the wrong kind of parameter.
|
||||
# To make debugging easier add an explicit check.
|
||||
if not isinstance(headers, httplib.HTTPMessage):
|
||||
raise TypeError(f"expected httplib.Message, got {type(headers)}.")
|
||||
raise TypeError("expected httplib.Message, got {0}.".format(type(headers)))
|
||||
|
||||
defects = getattr(headers, "defects", None)
|
||||
get_payload = getattr(headers, "get_payload", None)
|
||||
|
||||
unparsed_data = None
|
||||
if get_payload:
|
||||
# get_payload is actually email.message.Message.get_payload;
|
||||
# we're only interested in the result if it's not a multipart message
|
||||
if not headers.is_multipart():
|
||||
payload = get_payload()
|
||||
|
||||
# get_payload is actually email.message.Message.get_payload;
|
||||
# we're only interested in the result if it's not a multipart message
|
||||
if not headers.is_multipart():
|
||||
payload = headers.get_payload()
|
||||
if isinstance(payload, (bytes, str)):
|
||||
unparsed_data = payload
|
||||
if defects:
|
||||
# httplib is assuming a response body is available
|
||||
# when parsing headers even when httplib only sends
|
||||
# header data to parse_headers() This results in
|
||||
# defects on multipart responses in particular.
|
||||
# See: https://github.com/urllib3/urllib3/issues/800
|
||||
|
||||
if isinstance(payload, (bytes, str)):
|
||||
unparsed_data = payload
|
||||
|
||||
# httplib is assuming a response body is available
|
||||
# when parsing headers even when httplib only sends
|
||||
# header data to parse_headers() This results in
|
||||
# defects on multipart responses in particular.
|
||||
# See: https://github.com/urllib3/urllib3/issues/800
|
||||
|
||||
# So we ignore the following defects:
|
||||
# - StartBoundaryNotFoundDefect:
|
||||
# The claimed start boundary was never found.
|
||||
# - MultipartInvariantViolationDefect:
|
||||
# A message claimed to be a multipart but no subparts were found.
|
||||
defects = [
|
||||
defect
|
||||
for defect in headers.defects
|
||||
if not isinstance(
|
||||
defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect)
|
||||
)
|
||||
]
|
||||
# So we ignore the following defects:
|
||||
# - StartBoundaryNotFoundDefect:
|
||||
# The claimed start boundary was never found.
|
||||
# - MultipartInvariantViolationDefect:
|
||||
# A message claimed to be a multipart but no subparts were found.
|
||||
defects = [
|
||||
defect
|
||||
for defect in defects
|
||||
if not isinstance(
|
||||
defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect)
|
||||
)
|
||||
]
|
||||
|
||||
if defects or unparsed_data:
|
||||
raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)
|
||||
|
||||
|
||||
def is_response_to_head(response: httplib.HTTPResponse) -> bool:
|
||||
def is_response_to_head(response):
|
||||
"""
|
||||
Checks whether the request of a response has been a HEAD-request.
|
||||
Handles the quirks of AppEngine.
|
||||
|
||||
:param http.client.HTTPResponse response:
|
||||
Response to check if the originating request
|
||||
used 'HEAD' as a method.
|
||||
"""
|
||||
# FIXME: Can we do this somehow without accessing private httplib _method?
|
||||
method_str = response._method # type: str # type: ignore[attr-defined]
|
||||
return method_str.upper() == "HEAD"
|
||||
method = response._method
|
||||
if isinstance(method, int): # Platform-specific: Appengine
|
||||
return method == 3
|
||||
return method.upper() == "HEAD"
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import email
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import typing
|
||||
import warnings
|
||||
from collections import namedtuple
|
||||
from itertools import takewhile
|
||||
from types import TracebackType
|
||||
|
||||
from ..exceptions import (
|
||||
ConnectTimeoutError,
|
||||
@@ -18,51 +17,97 @@ from ..exceptions import (
|
||||
ReadTimeoutError,
|
||||
ResponseError,
|
||||
)
|
||||
from .util import reraise
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from typing_extensions import Self
|
||||
|
||||
from ..connectionpool import ConnectionPool
|
||||
from ..response import BaseHTTPResponse
|
||||
from ..packages import six
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Data structure for representing the metadata of requests that result in a retry.
|
||||
class RequestHistory(typing.NamedTuple):
|
||||
method: str | None
|
||||
url: str | None
|
||||
error: Exception | None
|
||||
status: int | None
|
||||
redirect_location: str | None
|
||||
RequestHistory = namedtuple(
|
||||
"RequestHistory", ["method", "url", "error", "status", "redirect_location"]
|
||||
)
|
||||
|
||||
|
||||
class Retry:
|
||||
# TODO: In v2 we can remove this sentinel and metaclass with deprecated options.
|
||||
_Default = object()
|
||||
|
||||
|
||||
class _RetryMeta(type):
|
||||
@property
|
||||
def DEFAULT_METHOD_WHITELIST(cls):
|
||||
warnings.warn(
|
||||
"Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
|
||||
"will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
return cls.DEFAULT_ALLOWED_METHODS
|
||||
|
||||
@DEFAULT_METHOD_WHITELIST.setter
|
||||
def DEFAULT_METHOD_WHITELIST(cls, value):
|
||||
warnings.warn(
|
||||
"Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
|
||||
"will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
cls.DEFAULT_ALLOWED_METHODS = value
|
||||
|
||||
@property
|
||||
def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls):
|
||||
warnings.warn(
|
||||
"Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
|
||||
"will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
|
||||
|
||||
@DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter
|
||||
def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value):
|
||||
warnings.warn(
|
||||
"Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
|
||||
"will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value
|
||||
|
||||
@property
|
||||
def BACKOFF_MAX(cls):
|
||||
warnings.warn(
|
||||
"Using 'Retry.BACKOFF_MAX' is deprecated and "
|
||||
"will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
return cls.DEFAULT_BACKOFF_MAX
|
||||
|
||||
@BACKOFF_MAX.setter
|
||||
def BACKOFF_MAX(cls, value):
|
||||
warnings.warn(
|
||||
"Using 'Retry.BACKOFF_MAX' is deprecated and "
|
||||
"will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
cls.DEFAULT_BACKOFF_MAX = value
|
||||
|
||||
|
||||
@six.add_metaclass(_RetryMeta)
|
||||
class Retry(object):
|
||||
"""Retry configuration.
|
||||
|
||||
Each retry attempt will create a new Retry object with updated values, so
|
||||
they can be safely reused.
|
||||
|
||||
Retries can be defined as a default for a pool:
|
||||
|
||||
.. code-block:: python
|
||||
Retries can be defined as a default for a pool::
|
||||
|
||||
retries = Retry(connect=5, read=2, redirect=5)
|
||||
http = PoolManager(retries=retries)
|
||||
response = http.request("GET", "https://example.com/")
|
||||
response = http.request('GET', 'http://example.com/')
|
||||
|
||||
Or per-request (which overrides the default for the pool):
|
||||
Or per-request (which overrides the default for the pool)::
|
||||
|
||||
.. code-block:: python
|
||||
response = http.request('GET', 'http://example.com/', retries=Retry(10))
|
||||
|
||||
response = http.request("GET", "https://example.com/", retries=Retry(10))
|
||||
Retries can be disabled by passing ``False``::
|
||||
|
||||
Retries can be disabled by passing ``False``:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
response = http.request("GET", "https://example.com/", retries=False)
|
||||
response = http.request('GET', 'http://example.com/', retries=False)
|
||||
|
||||
Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
|
||||
retries are disabled, in which case the causing exception will be raised.
|
||||
@@ -124,16 +169,21 @@ class Retry:
|
||||
If ``total`` is not set, it's a good idea to set this to 0 to account
|
||||
for unexpected edge cases and avoid infinite retry loops.
|
||||
|
||||
:param Collection allowed_methods:
|
||||
:param iterable allowed_methods:
|
||||
Set of uppercased HTTP method verbs that we should retry on.
|
||||
|
||||
By default, we only retry on methods which are considered to be
|
||||
idempotent (multiple requests with the same parameters end with the
|
||||
same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
|
||||
|
||||
Set to a ``None`` value to retry on any verb.
|
||||
Set to a ``False`` value to retry on any verb.
|
||||
|
||||
:param Collection status_forcelist:
|
||||
.. warning::
|
||||
|
||||
Previously this parameter was named ``method_whitelist``, that
|
||||
usage is deprecated in v1.26.0 and will be removed in v2.0.
|
||||
|
||||
:param iterable status_forcelist:
|
||||
A set of integer HTTP status codes that we should force a retry on.
|
||||
A retry is initiated if the request method is in ``allowed_methods``
|
||||
and the response status code is in ``status_forcelist``.
|
||||
@@ -145,25 +195,13 @@ class Retry:
|
||||
(most errors are resolved immediately by a second try without a
|
||||
delay). urllib3 will sleep for::
|
||||
|
||||
{backoff factor} * (2 ** ({number of previous retries}))
|
||||
{backoff factor} * (2 ** ({number of total retries} - 1))
|
||||
|
||||
seconds. If `backoff_jitter` is non-zero, this sleep is extended by::
|
||||
seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
|
||||
for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
|
||||
than :attr:`Retry.DEFAULT_BACKOFF_MAX`.
|
||||
|
||||
random.uniform(0, {backoff jitter})
|
||||
|
||||
seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will
|
||||
sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever
|
||||
be longer than `backoff_max`.
|
||||
|
||||
By default, backoff is disabled (factor set to 0).
|
||||
|
||||
:param float backoff_max:
|
||||
The maximum backoff time (in seconds) between retry attempts.
|
||||
This value caps the computed backoff from `backoff_factor`.
|
||||
|
||||
:param float backoff_jitter:
|
||||
Random jitter amount (in seconds) added to the computed backoff.
|
||||
Jitter is sampled uniformly from `0` to `backoff_jitter`.
|
||||
By default, backoff is disabled (set to 0).
|
||||
|
||||
:param bool raise_on_redirect: Whether, if the number of redirects is
|
||||
exhausted, to raise a MaxRetryError, or to return a response with a
|
||||
@@ -182,15 +220,10 @@ class Retry:
|
||||
Whether to respect Retry-After header on status codes defined as
|
||||
:attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
|
||||
|
||||
:param Collection remove_headers_on_redirect:
|
||||
:param iterable remove_headers_on_redirect:
|
||||
Sequence of headers to remove from the request when a response
|
||||
indicating a redirect is returned before firing off the redirected
|
||||
request.
|
||||
|
||||
:param int retry_after_max: Number of seconds to allow as the maximum for
|
||||
Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`.
|
||||
Any Retry-After headers larger than this value will be limited to this
|
||||
value.
|
||||
"""
|
||||
|
||||
#: Default methods to be used for ``allowed_methods``
|
||||
@@ -206,38 +239,48 @@ class Retry:
|
||||
["Cookie", "Authorization", "Proxy-Authorization"]
|
||||
)
|
||||
|
||||
#: Default maximum backoff time.
|
||||
#: Maximum backoff time.
|
||||
DEFAULT_BACKOFF_MAX = 120
|
||||
|
||||
# This is undocumented in the RFC. Setting to 6 hours matches other popular libraries.
|
||||
#: Default maximum allowed value for Retry-After headers in seconds
|
||||
DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600
|
||||
|
||||
# Backward compatibility; assigned outside of the class.
|
||||
DEFAULT: typing.ClassVar[Retry]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
total: bool | int | None = 10,
|
||||
connect: int | None = None,
|
||||
read: int | None = None,
|
||||
redirect: bool | int | None = None,
|
||||
status: int | None = None,
|
||||
other: int | None = None,
|
||||
allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
|
||||
status_forcelist: typing.Collection[int] | None = None,
|
||||
backoff_factor: float = 0,
|
||||
backoff_max: float = DEFAULT_BACKOFF_MAX,
|
||||
raise_on_redirect: bool = True,
|
||||
raise_on_status: bool = True,
|
||||
history: tuple[RequestHistory, ...] | None = None,
|
||||
respect_retry_after_header: bool = True,
|
||||
remove_headers_on_redirect: typing.Collection[
|
||||
str
|
||||
] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
|
||||
backoff_jitter: float = 0.0,
|
||||
retry_after_max: int = DEFAULT_RETRY_AFTER_MAX,
|
||||
) -> None:
|
||||
total=10,
|
||||
connect=None,
|
||||
read=None,
|
||||
redirect=None,
|
||||
status=None,
|
||||
other=None,
|
||||
allowed_methods=_Default,
|
||||
status_forcelist=None,
|
||||
backoff_factor=0,
|
||||
raise_on_redirect=True,
|
||||
raise_on_status=True,
|
||||
history=None,
|
||||
respect_retry_after_header=True,
|
||||
remove_headers_on_redirect=_Default,
|
||||
# TODO: Deprecated, remove in v2.0
|
||||
method_whitelist=_Default,
|
||||
):
|
||||
|
||||
if method_whitelist is not _Default:
|
||||
if allowed_methods is not _Default:
|
||||
raise ValueError(
|
||||
"Using both 'allowed_methods' and "
|
||||
"'method_whitelist' together is not allowed. "
|
||||
"Instead only use 'allowed_methods'"
|
||||
)
|
||||
warnings.warn(
|
||||
"Using 'method_whitelist' with Retry is deprecated and "
|
||||
"will be removed in v2.0. Use 'allowed_methods' instead",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
allowed_methods = method_whitelist
|
||||
if allowed_methods is _Default:
|
||||
allowed_methods = self.DEFAULT_ALLOWED_METHODS
|
||||
if remove_headers_on_redirect is _Default:
|
||||
remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
|
||||
|
||||
self.total = total
|
||||
self.connect = connect
|
||||
self.read = read
|
||||
@@ -252,18 +295,15 @@ class Retry:
|
||||
self.status_forcelist = status_forcelist or set()
|
||||
self.allowed_methods = allowed_methods
|
||||
self.backoff_factor = backoff_factor
|
||||
self.backoff_max = backoff_max
|
||||
self.retry_after_max = retry_after_max
|
||||
self.raise_on_redirect = raise_on_redirect
|
||||
self.raise_on_status = raise_on_status
|
||||
self.history = history or ()
|
||||
self.history = history or tuple()
|
||||
self.respect_retry_after_header = respect_retry_after_header
|
||||
self.remove_headers_on_redirect = frozenset(
|
||||
h.lower() for h in remove_headers_on_redirect
|
||||
[h.lower() for h in remove_headers_on_redirect]
|
||||
)
|
||||
self.backoff_jitter = backoff_jitter
|
||||
|
||||
def new(self, **kw: typing.Any) -> Self:
|
||||
def new(self, **kw):
|
||||
params = dict(
|
||||
total=self.total,
|
||||
connect=self.connect,
|
||||
@@ -271,29 +311,36 @@ class Retry:
|
||||
redirect=self.redirect,
|
||||
status=self.status,
|
||||
other=self.other,
|
||||
allowed_methods=self.allowed_methods,
|
||||
status_forcelist=self.status_forcelist,
|
||||
backoff_factor=self.backoff_factor,
|
||||
backoff_max=self.backoff_max,
|
||||
retry_after_max=self.retry_after_max,
|
||||
raise_on_redirect=self.raise_on_redirect,
|
||||
raise_on_status=self.raise_on_status,
|
||||
history=self.history,
|
||||
remove_headers_on_redirect=self.remove_headers_on_redirect,
|
||||
respect_retry_after_header=self.respect_retry_after_header,
|
||||
backoff_jitter=self.backoff_jitter,
|
||||
)
|
||||
|
||||
# TODO: If already given in **kw we use what's given to us
|
||||
# If not given we need to figure out what to pass. We decide
|
||||
# based on whether our class has the 'method_whitelist' property
|
||||
# and if so we pass the deprecated 'method_whitelist' otherwise
|
||||
# we use 'allowed_methods'. Remove in v2.0
|
||||
if "method_whitelist" not in kw and "allowed_methods" not in kw:
|
||||
if "method_whitelist" in self.__dict__:
|
||||
warnings.warn(
|
||||
"Using 'method_whitelist' with Retry is deprecated and "
|
||||
"will be removed in v2.0. Use 'allowed_methods' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
params["method_whitelist"] = self.allowed_methods
|
||||
else:
|
||||
params["allowed_methods"] = self.allowed_methods
|
||||
|
||||
params.update(kw)
|
||||
return type(self)(**params) # type: ignore[arg-type]
|
||||
return type(self)(**params)
|
||||
|
||||
@classmethod
|
||||
def from_int(
|
||||
cls,
|
||||
retries: Retry | bool | int | None,
|
||||
redirect: bool | int | None = True,
|
||||
default: Retry | bool | int | None = None,
|
||||
) -> Retry:
|
||||
def from_int(cls, retries, redirect=True, default=None):
|
||||
"""Backwards-compatibility for the old retries format."""
|
||||
if retries is None:
|
||||
retries = default if default is not None else cls.DEFAULT
|
||||
@@ -306,7 +353,7 @@ class Retry:
|
||||
log.debug("Converted retries value: %r -> %r", retries, new_retries)
|
||||
return new_retries
|
||||
|
||||
def get_backoff_time(self) -> float:
|
||||
def get_backoff_time(self):
|
||||
"""Formula for computing the current backoff
|
||||
|
||||
:rtype: float
|
||||
@@ -321,32 +368,32 @@ class Retry:
|
||||
return 0
|
||||
|
||||
backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
|
||||
if self.backoff_jitter != 0.0:
|
||||
backoff_value += random.random() * self.backoff_jitter
|
||||
return float(max(0, min(self.backoff_max, backoff_value)))
|
||||
return min(self.DEFAULT_BACKOFF_MAX, backoff_value)
|
||||
|
||||
def parse_retry_after(self, retry_after: str) -> float:
|
||||
seconds: float
|
||||
def parse_retry_after(self, retry_after):
|
||||
# Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
|
||||
if re.match(r"^\s*[0-9]+\s*$", retry_after):
|
||||
seconds = int(retry_after)
|
||||
else:
|
||||
retry_date_tuple = email.utils.parsedate_tz(retry_after)
|
||||
if retry_date_tuple is None:
|
||||
raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
|
||||
raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
|
||||
if retry_date_tuple[9] is None: # Python 2
|
||||
# Assume UTC if no timezone was specified
|
||||
# On Python2.7, parsedate_tz returns None for a timezone offset
|
||||
# instead of 0 if no timezone is given, where mktime_tz treats
|
||||
# a None timezone offset as local time.
|
||||
retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:]
|
||||
|
||||
retry_date = email.utils.mktime_tz(retry_date_tuple)
|
||||
seconds = retry_date - time.time()
|
||||
|
||||
seconds = max(seconds, 0)
|
||||
|
||||
# Check the seconds do not exceed the specified maximum
|
||||
if seconds > self.retry_after_max:
|
||||
seconds = self.retry_after_max
|
||||
if seconds < 0:
|
||||
seconds = 0
|
||||
|
||||
return seconds
|
||||
|
||||
def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
|
||||
def get_retry_after(self, response):
|
||||
"""Get the value of Retry-After in seconds."""
|
||||
|
||||
retry_after = response.headers.get("Retry-After")
|
||||
@@ -356,7 +403,7 @@ class Retry:
|
||||
|
||||
return self.parse_retry_after(retry_after)
|
||||
|
||||
def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
|
||||
def sleep_for_retry(self, response=None):
|
||||
retry_after = self.get_retry_after(response)
|
||||
if retry_after:
|
||||
time.sleep(retry_after)
|
||||
@@ -364,13 +411,13 @@ class Retry:
|
||||
|
||||
return False
|
||||
|
||||
def _sleep_backoff(self) -> None:
|
||||
def _sleep_backoff(self):
|
||||
backoff = self.get_backoff_time()
|
||||
if backoff <= 0:
|
||||
return
|
||||
time.sleep(backoff)
|
||||
|
||||
def sleep(self, response: BaseHTTPResponse | None = None) -> None:
|
||||
def sleep(self, response=None):
|
||||
"""Sleep between retry attempts.
|
||||
|
||||
This method will respect a server's ``Retry-After`` response header
|
||||
@@ -386,7 +433,7 @@ class Retry:
|
||||
|
||||
self._sleep_backoff()
|
||||
|
||||
def _is_connection_error(self, err: Exception) -> bool:
|
||||
def _is_connection_error(self, err):
|
||||
"""Errors when we're fairly sure that the server did not receive the
|
||||
request, so it should be safe to retry.
|
||||
"""
|
||||
@@ -394,23 +441,33 @@ class Retry:
|
||||
err = err.original_error
|
||||
return isinstance(err, ConnectTimeoutError)
|
||||
|
||||
def _is_read_error(self, err: Exception) -> bool:
|
||||
def _is_read_error(self, err):
|
||||
"""Errors that occur after the request has been started, so we should
|
||||
assume that the server began processing it.
|
||||
"""
|
||||
return isinstance(err, (ReadTimeoutError, ProtocolError))
|
||||
|
||||
def _is_method_retryable(self, method: str) -> bool:
|
||||
def _is_method_retryable(self, method):
|
||||
"""Checks if a given HTTP method should be retried upon, depending if
|
||||
it is included in the allowed_methods
|
||||
"""
|
||||
if self.allowed_methods and method.upper() not in self.allowed_methods:
|
||||
# TODO: For now favor if the Retry implementation sets its own method_whitelist
|
||||
# property outside of our constructor to avoid breaking custom implementations.
|
||||
if "method_whitelist" in self.__dict__:
|
||||
warnings.warn(
|
||||
"Using 'method_whitelist' with Retry is deprecated and "
|
||||
"will be removed in v2.0. Use 'allowed_methods' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
allowed_methods = self.method_whitelist
|
||||
else:
|
||||
allowed_methods = self.allowed_methods
|
||||
|
||||
if allowed_methods and method.upper() not in allowed_methods:
|
||||
return False
|
||||
return True
|
||||
|
||||
def is_retry(
|
||||
self, method: str, status_code: int, has_retry_after: bool = False
|
||||
) -> bool:
|
||||
def is_retry(self, method, status_code, has_retry_after=False):
|
||||
"""Is this method/status code retryable? (Based on allowlists and control
|
||||
variables such as the number of total retries to allow, whether to
|
||||
respect the Retry-After header, whether this header is present, and
|
||||
@@ -423,27 +480,24 @@ class Retry:
|
||||
if self.status_forcelist and status_code in self.status_forcelist:
|
||||
return True
|
||||
|
||||
return bool(
|
||||
return (
|
||||
self.total
|
||||
and self.respect_retry_after_header
|
||||
and has_retry_after
|
||||
and (status_code in self.RETRY_AFTER_STATUS_CODES)
|
||||
)
|
||||
|
||||
def is_exhausted(self) -> bool:
|
||||
def is_exhausted(self):
|
||||
"""Are we out of retries?"""
|
||||
retry_counts = [
|
||||
x
|
||||
for x in (
|
||||
self.total,
|
||||
self.connect,
|
||||
self.read,
|
||||
self.redirect,
|
||||
self.status,
|
||||
self.other,
|
||||
)
|
||||
if x
|
||||
]
|
||||
retry_counts = (
|
||||
self.total,
|
||||
self.connect,
|
||||
self.read,
|
||||
self.redirect,
|
||||
self.status,
|
||||
self.other,
|
||||
)
|
||||
retry_counts = list(filter(None, retry_counts))
|
||||
if not retry_counts:
|
||||
return False
|
||||
|
||||
@@ -451,18 +505,18 @@ class Retry:
|
||||
|
||||
def increment(
|
||||
self,
|
||||
method: str | None = None,
|
||||
url: str | None = None,
|
||||
response: BaseHTTPResponse | None = None,
|
||||
error: Exception | None = None,
|
||||
_pool: ConnectionPool | None = None,
|
||||
_stacktrace: TracebackType | None = None,
|
||||
) -> Self:
|
||||
method=None,
|
||||
url=None,
|
||||
response=None,
|
||||
error=None,
|
||||
_pool=None,
|
||||
_stacktrace=None,
|
||||
):
|
||||
"""Return a new Retry object with incremented retry counters.
|
||||
|
||||
:param response: A response object, or None, if the server did not
|
||||
return a response.
|
||||
:type response: :class:`~urllib3.response.BaseHTTPResponse`
|
||||
:type response: :class:`~urllib3.response.HTTPResponse`
|
||||
:param Exception error: An error encountered during the request, or
|
||||
None if the response was received successfully.
|
||||
|
||||
@@ -470,7 +524,7 @@ class Retry:
|
||||
"""
|
||||
if self.total is False and error:
|
||||
# Disabled, indicate to re-raise the error.
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
raise six.reraise(type(error), error, _stacktrace)
|
||||
|
||||
total = self.total
|
||||
if total is not None:
|
||||
@@ -488,14 +542,14 @@ class Retry:
|
||||
if error and self._is_connection_error(error):
|
||||
# Connect retry?
|
||||
if connect is False:
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
raise six.reraise(type(error), error, _stacktrace)
|
||||
elif connect is not None:
|
||||
connect -= 1
|
||||
|
||||
elif error and self._is_read_error(error):
|
||||
# Read retry?
|
||||
if read is False or method is None or not self._is_method_retryable(method):
|
||||
raise reraise(type(error), error, _stacktrace)
|
||||
if read is False or not self._is_method_retryable(method):
|
||||
raise six.reraise(type(error), error, _stacktrace)
|
||||
elif read is not None:
|
||||
read -= 1
|
||||
|
||||
@@ -509,9 +563,7 @@ class Retry:
|
||||
if redirect is not None:
|
||||
redirect -= 1
|
||||
cause = "too many redirects"
|
||||
response_redirect_location = response.get_redirect_location()
|
||||
if response_redirect_location:
|
||||
redirect_location = response_redirect_location
|
||||
redirect_location = response.get_redirect_location()
|
||||
status = response.status
|
||||
|
||||
else:
|
||||
@@ -539,18 +591,31 @@ class Retry:
|
||||
)
|
||||
|
||||
if new_retry.is_exhausted():
|
||||
reason = error or ResponseError(cause)
|
||||
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
|
||||
raise MaxRetryError(_pool, url, error or ResponseError(cause))
|
||||
|
||||
log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
|
||||
|
||||
return new_retry
|
||||
|
||||
def __repr__(self) -> str:
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
|
||||
f"read={self.read}, redirect={self.redirect}, status={self.status})"
|
||||
)
|
||||
"{cls.__name__}(total={self.total}, connect={self.connect}, "
|
||||
"read={self.read}, redirect={self.redirect}, status={self.status})"
|
||||
).format(cls=type(self), self=self)
|
||||
|
||||
def __getattr__(self, item):
|
||||
if item == "method_whitelist":
|
||||
# TODO: Remove this deprecated alias in v2.0
|
||||
warnings.warn(
|
||||
"Using 'method_whitelist' with Retry is deprecated and "
|
||||
"will be removed in v2.0. Use 'allowed_methods' instead",
|
||||
DeprecationWarning,
|
||||
)
|
||||
return self.allowed_methods
|
||||
try:
|
||||
return getattr(super(Retry, self), item)
|
||||
except AttributeError:
|
||||
return getattr(Retry, item)
|
||||
|
||||
|
||||
# For backwards compatibility (equivalent to pre-v1.9):
|
||||
|
||||
@@ -1,25 +1,28 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import absolute_import
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import typing
|
||||
import warnings
|
||||
from binascii import unhexlify
|
||||
from binascii import hexlify, unhexlify
|
||||
|
||||
from ..exceptions import ProxySchemeUnsupported, SSLError
|
||||
from .url import _BRACELESS_IPV6_ADDRZ_RE, _IPV4_RE
|
||||
from ..exceptions import (
|
||||
InsecurePlatformWarning,
|
||||
ProxySchemeUnsupported,
|
||||
SNIMissingWarning,
|
||||
SSLError,
|
||||
)
|
||||
from ..packages import six
|
||||
from .url import BRACELESS_IPV6_ADDRZ_RE, IPV4_RE
|
||||
|
||||
SSLContext = None
|
||||
SSLTransport = None
|
||||
HAS_NEVER_CHECK_COMMON_NAME = False
|
||||
HAS_SNI = False
|
||||
IS_PYOPENSSL = False
|
||||
IS_SECURETRANSPORT = False
|
||||
ALPN_PROTOCOLS = ["http/1.1"]
|
||||
|
||||
_TYPE_VERSION_INFO = tuple[int, int, int, str, int]
|
||||
|
||||
# Maps the length of a digest to a possible hash function producing this digest
|
||||
HASHFUNC_MAP = {
|
||||
length: getattr(hashlib, algorithm, None)
|
||||
@@ -27,84 +30,159 @@ HASHFUNC_MAP = {
|
||||
}
|
||||
|
||||
|
||||
def _is_has_never_check_common_name_reliable(
|
||||
openssl_version: str,
|
||||
) -> bool:
|
||||
# As of May 2023, all released versions of LibreSSL fail to reject certificates with
|
||||
# only common names, see https://github.com/urllib3/urllib3/pull/3024
|
||||
is_openssl = openssl_version.startswith("OpenSSL ")
|
||||
def _const_compare_digest_backport(a, b):
|
||||
"""
|
||||
Compare two digests of equal length in constant time.
|
||||
|
||||
return is_openssl
|
||||
The digests must be of type str/bytes.
|
||||
Returns True if the digests match, and False otherwise.
|
||||
"""
|
||||
result = abs(len(a) - len(b))
|
||||
for left, right in zip(bytearray(a), bytearray(b)):
|
||||
result |= left ^ right
|
||||
return result == 0
|
||||
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ssl import VerifyMode
|
||||
from typing import TypedDict
|
||||
_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport)
|
||||
|
||||
from .ssltransport import SSLTransport as SSLTransportType
|
||||
|
||||
class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False):
|
||||
subjectAltName: tuple[tuple[str, str], ...]
|
||||
subject: tuple[tuple[tuple[str, str], ...], ...]
|
||||
serialNumber: str
|
||||
|
||||
|
||||
# Mapping from 'ssl.PROTOCOL_TLSX' to 'TLSVersion.X'
|
||||
_SSL_VERSION_TO_TLS_VERSION: dict[int, int] = {}
|
||||
|
||||
try: # Do we have ssl at all?
|
||||
try: # Test for SSL features
|
||||
import ssl
|
||||
from ssl import ( # type: ignore[assignment]
|
||||
CERT_REQUIRED,
|
||||
HAS_NEVER_CHECK_COMMON_NAME,
|
||||
OP_NO_COMPRESSION,
|
||||
OP_NO_TICKET,
|
||||
OPENSSL_VERSION,
|
||||
PROTOCOL_TLS,
|
||||
PROTOCOL_TLS_CLIENT,
|
||||
VERIFY_X509_PARTIAL_CHAIN,
|
||||
VERIFY_X509_STRICT,
|
||||
OP_NO_SSLv2,
|
||||
OP_NO_SSLv3,
|
||||
SSLContext,
|
||||
TLSVersion,
|
||||
)
|
||||
from ssl import CERT_REQUIRED, wrap_socket
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from ssl import HAS_SNI # Has SNI?
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from .ssltransport import SSLTransport
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
try: # Platform-specific: Python 3.6
|
||||
from ssl import PROTOCOL_TLS
|
||||
|
||||
PROTOCOL_SSLv23 = PROTOCOL_TLS
|
||||
|
||||
# Setting SSLContext.hostname_checks_common_name = False didn't work with
|
||||
# LibreSSL, check details in the used function.
|
||||
if HAS_NEVER_CHECK_COMMON_NAME and not _is_has_never_check_common_name_reliable(
|
||||
OPENSSL_VERSION,
|
||||
): # Defensive:
|
||||
HAS_NEVER_CHECK_COMMON_NAME = False
|
||||
|
||||
# Need to be careful here in case old TLS versions get
|
||||
# removed in future 'ssl' module implementations.
|
||||
for attr in ("TLSv1", "TLSv1_1", "TLSv1_2"):
|
||||
try:
|
||||
_SSL_VERSION_TO_TLS_VERSION[getattr(ssl, f"PROTOCOL_{attr}")] = getattr(
|
||||
TLSVersion, attr
|
||||
)
|
||||
except AttributeError: # Defensive:
|
||||
continue
|
||||
|
||||
from .ssltransport import SSLTransport # type: ignore[assignment]
|
||||
except ImportError:
|
||||
OP_NO_COMPRESSION = 0x20000 # type: ignore[assignment, misc]
|
||||
OP_NO_TICKET = 0x4000 # type: ignore[assignment, misc]
|
||||
OP_NO_SSLv2 = 0x1000000 # type: ignore[assignment, misc]
|
||||
OP_NO_SSLv3 = 0x2000000 # type: ignore[assignment, misc]
|
||||
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 # type: ignore[assignment, misc]
|
||||
PROTOCOL_TLS_CLIENT = 16 # type: ignore[assignment, misc]
|
||||
VERIFY_X509_PARTIAL_CHAIN = 0x80000 # type: ignore[assignment,misc]
|
||||
VERIFY_X509_STRICT = 0x20 # type: ignore[assignment, misc]
|
||||
try:
|
||||
from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
|
||||
|
||||
PROTOCOL_SSLv23 = PROTOCOL_TLS
|
||||
except ImportError:
|
||||
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
|
||||
|
||||
try:
|
||||
from ssl import PROTOCOL_TLS_CLIENT
|
||||
except ImportError:
|
||||
PROTOCOL_TLS_CLIENT = PROTOCOL_TLS
|
||||
|
||||
|
||||
_TYPE_PEER_CERT_RET = typing.Union["_TYPE_PEER_CERT_RET_DICT", bytes, None]
|
||||
try:
|
||||
from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3
|
||||
except ImportError:
|
||||
OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
|
||||
OP_NO_COMPRESSION = 0x20000
|
||||
|
||||
|
||||
def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
|
||||
try: # OP_NO_TICKET was added in Python 3.6
|
||||
from ssl import OP_NO_TICKET
|
||||
except ImportError:
|
||||
OP_NO_TICKET = 0x4000
|
||||
|
||||
|
||||
# A secure default.
|
||||
# Sources for more information on TLS ciphers:
|
||||
#
|
||||
# - https://wiki.mozilla.org/Security/Server_Side_TLS
|
||||
# - https://www.ssllabs.com/projects/best-practices/index.html
|
||||
# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
|
||||
#
|
||||
# The general intent is:
|
||||
# - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
|
||||
# - prefer ECDHE over DHE for better performance,
|
||||
# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and
|
||||
# security,
|
||||
# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common,
|
||||
# - disable NULL authentication, MD5 MACs, DSS, and other
|
||||
# insecure ciphers for security reasons.
|
||||
# - NOTE: TLS 1.3 cipher suites are managed through a different interface
|
||||
# not exposed by CPython (yet!) and are enabled by default if they're available.
|
||||
DEFAULT_CIPHERS = ":".join(
|
||||
[
|
||||
"ECDHE+AESGCM",
|
||||
"ECDHE+CHACHA20",
|
||||
"DHE+AESGCM",
|
||||
"DHE+CHACHA20",
|
||||
"ECDH+AESGCM",
|
||||
"DH+AESGCM",
|
||||
"ECDH+AES",
|
||||
"DH+AES",
|
||||
"RSA+AESGCM",
|
||||
"RSA+AES",
|
||||
"!aNULL",
|
||||
"!eNULL",
|
||||
"!MD5",
|
||||
"!DSS",
|
||||
]
|
||||
)
|
||||
|
||||
try:
|
||||
from ssl import SSLContext # Modern SSL?
|
||||
except ImportError:
|
||||
|
||||
class SSLContext(object): # Platform-specific: Python 2
|
||||
def __init__(self, protocol_version):
|
||||
self.protocol = protocol_version
|
||||
# Use default values from a real SSLContext
|
||||
self.check_hostname = False
|
||||
self.verify_mode = ssl.CERT_NONE
|
||||
self.ca_certs = None
|
||||
self.options = 0
|
||||
self.certfile = None
|
||||
self.keyfile = None
|
||||
self.ciphers = None
|
||||
|
||||
def load_cert_chain(self, certfile, keyfile):
|
||||
self.certfile = certfile
|
||||
self.keyfile = keyfile
|
||||
|
||||
def load_verify_locations(self, cafile=None, capath=None, cadata=None):
|
||||
self.ca_certs = cafile
|
||||
|
||||
if capath is not None:
|
||||
raise SSLError("CA directories not supported in older Pythons")
|
||||
|
||||
if cadata is not None:
|
||||
raise SSLError("CA data not supported in older Pythons")
|
||||
|
||||
def set_ciphers(self, cipher_suite):
|
||||
self.ciphers = cipher_suite
|
||||
|
||||
def wrap_socket(self, socket, server_hostname=None, server_side=False):
|
||||
warnings.warn(
|
||||
"A true SSLContext object is not available. This prevents "
|
||||
"urllib3 from configuring SSL appropriately and may cause "
|
||||
"certain SSL connections to fail. You can upgrade to a newer "
|
||||
"version of Python to solve this. For more information, see "
|
||||
"https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"
|
||||
"#ssl-warnings",
|
||||
InsecurePlatformWarning,
|
||||
)
|
||||
kwargs = {
|
||||
"keyfile": self.keyfile,
|
||||
"certfile": self.certfile,
|
||||
"ca_certs": self.ca_certs,
|
||||
"cert_reqs": self.verify_mode,
|
||||
"ssl_version": self.protocol,
|
||||
"server_side": server_side,
|
||||
}
|
||||
return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
|
||||
|
||||
|
||||
def assert_fingerprint(cert, fingerprint):
|
||||
"""
|
||||
Checks if given fingerprint matches the supplied certificate.
|
||||
|
||||
@@ -114,17 +192,16 @@ def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
|
||||
Fingerprint as string of hexdigits, can be interspersed by colons.
|
||||
"""
|
||||
|
||||
if cert is None:
|
||||
raise SSLError("No certificate for the peer.")
|
||||
|
||||
fingerprint = fingerprint.replace(":", "").lower()
|
||||
digest_length = len(fingerprint)
|
||||
if digest_length not in HASHFUNC_MAP:
|
||||
raise SSLError(f"Fingerprint of invalid length: {fingerprint}")
|
||||
raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint))
|
||||
hashfunc = HASHFUNC_MAP.get(digest_length)
|
||||
if hashfunc is None:
|
||||
raise SSLError(
|
||||
f"Hash function implementation unavailable for fingerprint length: {digest_length}"
|
||||
"Hash function implementation unavailable for fingerprint length: {0}".format(
|
||||
digest_length
|
||||
)
|
||||
)
|
||||
|
||||
# We need encode() here for py32; works on py2 and p33.
|
||||
@@ -132,13 +209,15 @@ def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
|
||||
|
||||
cert_digest = hashfunc(cert).digest()
|
||||
|
||||
if not hmac.compare_digest(cert_digest, fingerprint_bytes):
|
||||
if not _const_compare_digest(cert_digest, fingerprint_bytes):
|
||||
raise SSLError(
|
||||
f'Fingerprints did not match. Expected "{fingerprint}", got "{cert_digest.hex()}"'
|
||||
'Fingerprints did not match. Expected "{0}", got "{1}".'.format(
|
||||
fingerprint, hexlify(cert_digest)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode:
|
||||
def resolve_cert_reqs(candidate):
|
||||
"""
|
||||
Resolves the argument to a numeric constant, which can be passed to
|
||||
the wrap_socket function/method from the ssl module.
|
||||
@@ -156,12 +235,12 @@ def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode:
|
||||
res = getattr(ssl, candidate, None)
|
||||
if res is None:
|
||||
res = getattr(ssl, "CERT_" + candidate)
|
||||
return res # type: ignore[no-any-return]
|
||||
return res
|
||||
|
||||
return candidate # type: ignore[return-value]
|
||||
return candidate
|
||||
|
||||
|
||||
def resolve_ssl_version(candidate: None | int | str) -> int:
|
||||
def resolve_ssl_version(candidate):
|
||||
"""
|
||||
like resolve_cert_reqs
|
||||
"""
|
||||
@@ -172,34 +251,35 @@ def resolve_ssl_version(candidate: None | int | str) -> int:
|
||||
res = getattr(ssl, candidate, None)
|
||||
if res is None:
|
||||
res = getattr(ssl, "PROTOCOL_" + candidate)
|
||||
return typing.cast(int, res)
|
||||
return res
|
||||
|
||||
return candidate
|
||||
|
||||
|
||||
def create_urllib3_context(
|
||||
ssl_version: int | None = None,
|
||||
cert_reqs: int | None = None,
|
||||
options: int | None = None,
|
||||
ciphers: str | None = None,
|
||||
ssl_minimum_version: int | None = None,
|
||||
ssl_maximum_version: int | None = None,
|
||||
verify_flags: int | None = None,
|
||||
) -> ssl.SSLContext:
|
||||
"""Creates and configures an :class:`ssl.SSLContext` instance for use with urllib3.
|
||||
ssl_version=None, cert_reqs=None, options=None, ciphers=None
|
||||
):
|
||||
"""All arguments have the same meaning as ``ssl_wrap_socket``.
|
||||
|
||||
By default, this function does a lot of the same work that
|
||||
``ssl.create_default_context`` does on Python 3.4+. It:
|
||||
|
||||
- Disables SSLv2, SSLv3, and compression
|
||||
- Sets a restricted set of server ciphers
|
||||
|
||||
If you wish to enable SSLv3, you can do::
|
||||
|
||||
from urllib3.util import ssl_
|
||||
context = ssl_.create_urllib3_context()
|
||||
context.options &= ~ssl_.OP_NO_SSLv3
|
||||
|
||||
You can do the same to enable compression (substituting ``COMPRESSION``
|
||||
for ``SSLv3`` in the last line above).
|
||||
|
||||
:param ssl_version:
|
||||
The desired protocol version to use. This will default to
|
||||
PROTOCOL_SSLv23 which will negotiate the highest protocol that both
|
||||
the server and your installation of OpenSSL support.
|
||||
|
||||
This parameter is deprecated instead use 'ssl_minimum_version'.
|
||||
:param ssl_minimum_version:
|
||||
The minimum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
|
||||
:param ssl_maximum_version:
|
||||
The maximum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
|
||||
Not recommended to set to anything other than 'ssl.TLSVersion.MAXIMUM_SUPPORTED' which is the
|
||||
default value.
|
||||
:param cert_reqs:
|
||||
Whether to require the certificate verification. This defaults to
|
||||
``ssl.CERT_REQUIRED``.
|
||||
@@ -207,61 +287,18 @@ def create_urllib3_context(
|
||||
Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
|
||||
``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``.
|
||||
:param ciphers:
|
||||
Which cipher suites to allow the server to select. Defaults to either system configured
|
||||
ciphers if OpenSSL 1.1.1+, otherwise uses a secure default set of ciphers.
|
||||
:param verify_flags:
|
||||
The flags for certificate verification operations. These default to
|
||||
``ssl.VERIFY_X509_PARTIAL_CHAIN`` and ``ssl.VERIFY_X509_STRICT`` for Python 3.13+.
|
||||
Which cipher suites to allow the server to select.
|
||||
:returns:
|
||||
Constructed SSLContext object with specified options
|
||||
:rtype: SSLContext
|
||||
"""
|
||||
if SSLContext is None:
|
||||
raise TypeError("Can't create an SSLContext object without an ssl module")
|
||||
# PROTOCOL_TLS is deprecated in Python 3.10
|
||||
if not ssl_version or ssl_version == PROTOCOL_TLS:
|
||||
ssl_version = PROTOCOL_TLS_CLIENT
|
||||
|
||||
# This means 'ssl_version' was specified as an exact value.
|
||||
if ssl_version not in (None, PROTOCOL_TLS, PROTOCOL_TLS_CLIENT):
|
||||
# Disallow setting 'ssl_version' and 'ssl_minimum|maximum_version'
|
||||
# to avoid conflicts.
|
||||
if ssl_minimum_version is not None or ssl_maximum_version is not None:
|
||||
raise ValueError(
|
||||
"Can't specify both 'ssl_version' and either "
|
||||
"'ssl_minimum_version' or 'ssl_maximum_version'"
|
||||
)
|
||||
context = SSLContext(ssl_version)
|
||||
|
||||
# 'ssl_version' is deprecated and will be removed in the future.
|
||||
else:
|
||||
# Use 'ssl_minimum_version' and 'ssl_maximum_version' instead.
|
||||
ssl_minimum_version = _SSL_VERSION_TO_TLS_VERSION.get(
|
||||
ssl_version, TLSVersion.MINIMUM_SUPPORTED
|
||||
)
|
||||
ssl_maximum_version = _SSL_VERSION_TO_TLS_VERSION.get(
|
||||
ssl_version, TLSVersion.MAXIMUM_SUPPORTED
|
||||
)
|
||||
|
||||
# This warning message is pushing users to use 'ssl_minimum_version'
|
||||
# instead of both min/max. Best practice is to only set the minimum version and
|
||||
# keep the maximum version to be it's default value: 'TLSVersion.MAXIMUM_SUPPORTED'
|
||||
warnings.warn(
|
||||
"'ssl_version' option is deprecated and will be "
|
||||
"removed in urllib3 v3.0. Instead use 'ssl_minimum_version'",
|
||||
category=FutureWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
context = SSLContext(PROTOCOL_TLS_CLIENT)
|
||||
if ssl_minimum_version is not None:
|
||||
context.minimum_version = ssl_minimum_version
|
||||
else: # pyOpenSSL defaults to 'MINIMUM_SUPPORTED' so explicitly set TLSv1.2 here
|
||||
context.minimum_version = TLSVersion.TLSv1_2
|
||||
|
||||
if ssl_maximum_version is not None:
|
||||
context.maximum_version = ssl_maximum_version
|
||||
|
||||
# Unless we're given ciphers defer to either system ciphers in
|
||||
# the case of OpenSSL 1.1.1+ or use our own secure default ciphers.
|
||||
if ciphers:
|
||||
context.set_ciphers(ciphers)
|
||||
context.set_ciphers(ciphers or DEFAULT_CIPHERS)
|
||||
|
||||
# Setting the default here, as we may have no ssl module on import
|
||||
cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
|
||||
@@ -283,103 +320,65 @@ def create_urllib3_context(
|
||||
|
||||
context.options |= options
|
||||
|
||||
if verify_flags is None:
|
||||
verify_flags = 0
|
||||
# In Python 3.13+ ssl.create_default_context() sets VERIFY_X509_PARTIAL_CHAIN
|
||||
# and VERIFY_X509_STRICT so we do the same
|
||||
if sys.version_info >= (3, 13):
|
||||
verify_flags |= VERIFY_X509_PARTIAL_CHAIN
|
||||
verify_flags |= VERIFY_X509_STRICT
|
||||
|
||||
context.verify_flags |= verify_flags
|
||||
|
||||
# Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is
|
||||
# necessary for conditional client cert authentication with TLS 1.3.
|
||||
# The attribute is None for OpenSSL <= 1.1.0 or does not exist when using
|
||||
# an SSLContext created by pyOpenSSL.
|
||||
if getattr(context, "post_handshake_auth", None) is not None:
|
||||
# The attribute is None for OpenSSL <= 1.1.0 or does not exist in older
|
||||
# versions of Python. We only enable on Python 3.7.4+ or if certificate
|
||||
# verification is enabled to work around Python issue #37428
|
||||
# See: https://bugs.python.org/issue37428
|
||||
if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr(
|
||||
context, "post_handshake_auth", None
|
||||
) is not None:
|
||||
context.post_handshake_auth = True
|
||||
|
||||
def disable_check_hostname():
|
||||
if (
|
||||
getattr(context, "check_hostname", None) is not None
|
||||
): # Platform-specific: Python 3.2
|
||||
# We do our own verification, including fingerprints and alternative
|
||||
# hostnames. So disable it here
|
||||
context.check_hostname = False
|
||||
|
||||
# The order of the below lines setting verify_mode and check_hostname
|
||||
# matter due to safe-guards SSLContext has to prevent an SSLContext with
|
||||
# check_hostname=True, verify_mode=NONE/OPTIONAL.
|
||||
# We always set 'check_hostname=False' for pyOpenSSL so we rely on our own
|
||||
# 'ssl.match_hostname()' implementation.
|
||||
if cert_reqs == ssl.CERT_REQUIRED and not IS_PYOPENSSL:
|
||||
# check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more
|
||||
# complex because we don't know whether PROTOCOL_TLS_CLIENT will be used
|
||||
# or not so we don't know the initial state of the freshly created SSLContext.
|
||||
if cert_reqs == ssl.CERT_REQUIRED:
|
||||
context.verify_mode = cert_reqs
|
||||
context.check_hostname = True
|
||||
disable_check_hostname()
|
||||
else:
|
||||
context.check_hostname = False
|
||||
disable_check_hostname()
|
||||
context.verify_mode = cert_reqs
|
||||
|
||||
context.hostname_checks_common_name = False
|
||||
|
||||
if "SSLKEYLOGFILE" in os.environ:
|
||||
sslkeylogfile = os.path.expandvars(os.environ.get("SSLKEYLOGFILE"))
|
||||
else:
|
||||
sslkeylogfile = None
|
||||
if sslkeylogfile:
|
||||
context.keylog_filename = sslkeylogfile
|
||||
# Enable logging of TLS session keys via defacto standard environment variable
|
||||
# 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values.
|
||||
if hasattr(context, "keylog_filename"):
|
||||
sslkeylogfile = os.environ.get("SSLKEYLOGFILE")
|
||||
if sslkeylogfile:
|
||||
context.keylog_filename = sslkeylogfile
|
||||
|
||||
return context
|
||||
|
||||
|
||||
@typing.overload
|
||||
def ssl_wrap_socket(
|
||||
sock: socket.socket,
|
||||
keyfile: str | None = ...,
|
||||
certfile: str | None = ...,
|
||||
cert_reqs: int | None = ...,
|
||||
ca_certs: str | None = ...,
|
||||
server_hostname: str | None = ...,
|
||||
ssl_version: int | None = ...,
|
||||
ciphers: str | None = ...,
|
||||
ssl_context: ssl.SSLContext | None = ...,
|
||||
ca_cert_dir: str | None = ...,
|
||||
key_password: str | None = ...,
|
||||
ca_cert_data: None | str | bytes = ...,
|
||||
tls_in_tls: typing.Literal[False] = ...,
|
||||
) -> ssl.SSLSocket: ...
|
||||
|
||||
|
||||
@typing.overload
|
||||
def ssl_wrap_socket(
|
||||
sock: socket.socket,
|
||||
keyfile: str | None = ...,
|
||||
certfile: str | None = ...,
|
||||
cert_reqs: int | None = ...,
|
||||
ca_certs: str | None = ...,
|
||||
server_hostname: str | None = ...,
|
||||
ssl_version: int | None = ...,
|
||||
ciphers: str | None = ...,
|
||||
ssl_context: ssl.SSLContext | None = ...,
|
||||
ca_cert_dir: str | None = ...,
|
||||
key_password: str | None = ...,
|
||||
ca_cert_data: None | str | bytes = ...,
|
||||
tls_in_tls: bool = ...,
|
||||
) -> ssl.SSLSocket | SSLTransportType: ...
|
||||
|
||||
|
||||
def ssl_wrap_socket(
|
||||
sock: socket.socket,
|
||||
keyfile: str | None = None,
|
||||
certfile: str | None = None,
|
||||
cert_reqs: int | None = None,
|
||||
ca_certs: str | None = None,
|
||||
server_hostname: str | None = None,
|
||||
ssl_version: int | None = None,
|
||||
ciphers: str | None = None,
|
||||
ssl_context: ssl.SSLContext | None = None,
|
||||
ca_cert_dir: str | None = None,
|
||||
key_password: str | None = None,
|
||||
ca_cert_data: None | str | bytes = None,
|
||||
tls_in_tls: bool = False,
|
||||
) -> ssl.SSLSocket | SSLTransportType:
|
||||
sock,
|
||||
keyfile=None,
|
||||
certfile=None,
|
||||
cert_reqs=None,
|
||||
ca_certs=None,
|
||||
server_hostname=None,
|
||||
ssl_version=None,
|
||||
ciphers=None,
|
||||
ssl_context=None,
|
||||
ca_cert_dir=None,
|
||||
key_password=None,
|
||||
ca_cert_data=None,
|
||||
tls_in_tls=False,
|
||||
):
|
||||
"""
|
||||
All arguments except for server_hostname, ssl_context, tls_in_tls, ca_cert_data and
|
||||
ca_cert_dir have the same meaning as they do when using
|
||||
:func:`ssl.create_default_context`, :meth:`ssl.SSLContext.load_cert_chain`,
|
||||
:meth:`ssl.SSLContext.set_ciphers` and :meth:`ssl.SSLContext.wrap_socket`.
|
||||
All arguments except for server_hostname, ssl_context, and ca_cert_dir have
|
||||
the same meaning as they do when using :func:`ssl.wrap_socket`.
|
||||
|
||||
:param server_hostname:
|
||||
When SNI is supported, the expected hostname of the certificate
|
||||
@@ -402,18 +401,19 @@ def ssl_wrap_socket(
|
||||
"""
|
||||
context = ssl_context
|
||||
if context is None:
|
||||
# Note: This branch of code and all the variables in it are only used in tests.
|
||||
# We should consider deprecating and removing this code.
|
||||
# Note: This branch of code and all the variables in it are no longer
|
||||
# used by urllib3 itself. We should consider deprecating and removing
|
||||
# this code.
|
||||
context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers)
|
||||
|
||||
if ca_certs or ca_cert_dir or ca_cert_data:
|
||||
try:
|
||||
context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data)
|
||||
except OSError as e:
|
||||
raise SSLError(e) from e
|
||||
except (IOError, OSError) as e:
|
||||
raise SSLError(e)
|
||||
|
||||
elif ssl_context is None and hasattr(context, "load_default_certs"):
|
||||
# try to load OS default certs; works well on Windows.
|
||||
# try to load OS default certs; works well on Windows (require Python3.4+)
|
||||
context.load_default_certs()
|
||||
|
||||
# Attempt to detect if we get the goofy behavior of the
|
||||
@@ -428,28 +428,57 @@ def ssl_wrap_socket(
|
||||
else:
|
||||
context.load_cert_chain(certfile, keyfile, key_password)
|
||||
|
||||
context.set_alpn_protocols(ALPN_PROTOCOLS)
|
||||
try:
|
||||
if hasattr(context, "set_alpn_protocols"):
|
||||
context.set_alpn_protocols(ALPN_PROTOCOLS)
|
||||
except NotImplementedError: # Defensive: in CI, we always have set_alpn_protocols
|
||||
pass
|
||||
|
||||
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
|
||||
# If we detect server_hostname is an IP address then the SNI
|
||||
# extension should not be used according to RFC3546 Section 3.1
|
||||
use_sni_hostname = server_hostname and not is_ipaddress(server_hostname)
|
||||
# SecureTransport uses server_hostname in certificate verification.
|
||||
send_sni = (use_sni_hostname and HAS_SNI) or (
|
||||
IS_SECURETRANSPORT and server_hostname
|
||||
)
|
||||
# Do not warn the user if server_hostname is an invalid SNI hostname.
|
||||
if not HAS_SNI and use_sni_hostname:
|
||||
warnings.warn(
|
||||
"An HTTPS request has been made, but the SNI (Server Name "
|
||||
"Indication) extension to TLS is not available on this platform. "
|
||||
"This may cause the server to present an incorrect TLS "
|
||||
"certificate, which can cause validation failures. You can upgrade to "
|
||||
"a newer version of Python to solve this. For more information, see "
|
||||
"https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"
|
||||
"#ssl-warnings",
|
||||
SNIMissingWarning,
|
||||
)
|
||||
|
||||
if send_sni:
|
||||
ssl_sock = _ssl_wrap_socket_impl(
|
||||
sock, context, tls_in_tls, server_hostname=server_hostname
|
||||
)
|
||||
else:
|
||||
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls)
|
||||
return ssl_sock
|
||||
|
||||
|
||||
def is_ipaddress(hostname: str | bytes) -> bool:
|
||||
def is_ipaddress(hostname):
|
||||
"""Detects whether the hostname given is an IPv4 or IPv6 address.
|
||||
Also detects IPv6 addresses with Zone IDs.
|
||||
|
||||
:param str hostname: Hostname to examine.
|
||||
:return: True if the hostname is an IP address, False otherwise.
|
||||
"""
|
||||
if isinstance(hostname, bytes):
|
||||
if not six.PY2 and isinstance(hostname, bytes):
|
||||
# IDN A-label bytes are ASCII compatible.
|
||||
hostname = hostname.decode("ascii")
|
||||
return bool(_IPV4_RE.match(hostname) or _BRACELESS_IPV6_ADDRZ_RE.match(hostname))
|
||||
return bool(IPV4_RE.match(hostname) or BRACELESS_IPV6_ADDRZ_RE.match(hostname))
|
||||
|
||||
|
||||
def _is_key_file_encrypted(key_file: str) -> bool:
|
||||
def _is_key_file_encrypted(key_file):
|
||||
"""Detects if a key file is encrypted or not."""
|
||||
with open(key_file) as f:
|
||||
with open(key_file, "r") as f:
|
||||
for line in f:
|
||||
# Look for Proc-Type: 4,ENCRYPTED
|
||||
if "ENCRYPTED" in line:
|
||||
@@ -458,12 +487,7 @@ def _is_key_file_encrypted(key_file: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _ssl_wrap_socket_impl(
|
||||
sock: socket.socket,
|
||||
ssl_context: ssl.SSLContext,
|
||||
tls_in_tls: bool,
|
||||
server_hostname: str | None = None,
|
||||
) -> ssl.SSLSocket | SSLTransportType:
|
||||
def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None):
|
||||
if tls_in_tls:
|
||||
if not SSLTransport:
|
||||
# Import error, ssl is not available.
|
||||
@@ -474,4 +498,7 @@ def _ssl_wrap_socket_impl(
|
||||
SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context)
|
||||
return SSLTransport(sock, ssl_context, server_hostname)
|
||||
|
||||
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
|
||||
if server_hostname:
|
||||
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
|
||||
else:
|
||||
return ssl_context.wrap_socket(sock)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user