Compare commits

...

3 Commits

Author SHA1 Message Date
Ninluc 2dd664c4b4 UART & Sensors
Build, push image, and notify Watchtower / build-image (push) Successful in 1m39s
Build, push image, and notify Watchtower / notify (push) Successful in 1m43s
2026-07-23 15:50:43 +02:00
Ninluc 1ec3ee7abf Move requirement file 2026-07-22 16:47:10 +02:00
Ninluc dd808b1d68 Ignore venv 2026-07-20 14:21:41 +02:00
106 changed files with 5288 additions and 7422 deletions
+1
View File
@@ -1,2 +1,3 @@
__pycache__/
orchestrateur/db.sqlite-shm
venv/
+13 -10
View File
@@ -20,9 +20,9 @@ db = client["microwave_network_db"]
cooking_collection = db["cooking_parameters"]
device_network_collection = db["device_network"]
# Ensure the photo storage directory exists when the app starts
PHOTO_DIR = "storage/dishPhotos"
os.makedirs(PHOTO_DIR, exist_ok=True)
# Ensure the camera image storage directory exists when the app starts
CAMERA_IMAGE_DIR = "storage/dishCameraImages"
os.makedirs(CAMERA_IMAGE_DIR, exist_ok=True)
# ---------------------------------------------------------
# Routes
@@ -40,24 +40,24 @@ def cooking_params():
if not data:
return jsonify({"error": "Invalid or missing JSON payload"}), 400
# 1. Handle the Photo
photo_b64 = data.get("photo")
if photo_b64:
# 1. Handle the Camera Image
camera_image_b64 = data.get("camera_image")
if camera_image_b64:
# Generate a unique filename using UUID to avoid overwriting
filename = f"dish_{uuid.uuid4().hex}.jpg"
filepath = os.path.join(PHOTO_DIR, filename)
filepath = os.path.join(CAMERA_IMAGE_DIR, filename)
try:
# Decode the base64 string and save it as a binary file
with open(filepath, "wb") as f:
f.write(base64.b64decode(photo_b64))
f.write(base64.b64decode(camera_image_b64))
# Replace the giant base64 string in the dictionary with the local file path
# so we don't bloat the MongoDB document
data["photo"] = filepath
data["camera_image"] = filepath
except Exception as e:
return jsonify({"error": f"Failed to save photo: {str(e)}"}), 500
return jsonify({"error": f"Failed to save camera image: {str(e)}"}), 500
# 2. Save to MongoDB
try:
@@ -70,6 +70,9 @@ def cooking_params():
except Exception as e:
return jsonify({"error": f"Database error: {str(e)}"}), 500
# 3. Returns with the cooking parameters
@app.route("/device-network", methods=["POST"])
+5 -2
View File
@@ -14,7 +14,7 @@ RPI_SYSTEMD_SERVICE="smartwave.service"
# Vérification des arguments
if [ -z "$1" ]; then
echo "Usage: ./deploy.sh [wifi|lora|rpi|all]"
echo "Usage: ./deploy.sh [wifi|mqtt|lora|rpi|all]"
exit 1
fi
@@ -79,6 +79,9 @@ case $CIBLE in
"wifi")
deploy_to_esp "micro_ondes/esp_wifi" "$PORT_ESP_WIFI" "ESP-WIFI"
;;
"mqtt")
deploy_to_esp "micro_ondes/esp_wifi" "$PORT_ESP_WIFI" "ESP-WIFI"
;;
"lora")
deploy_to_esp "micro_ondes/esp_lora" "$PORT_ESP_LORA" "ESP-LORA"
;;
@@ -92,6 +95,6 @@ case $CIBLE in
# Ajoute les autres ici
;;
*)
echo "Cible inconnue. Utilise 'wifi', 'lora' ou 'all'."
echo "Cible inconnue. Utilise 'wifi', 'lora', 'mqtt' ou 'all'."
;;
esac
+12 -1
View File
@@ -1,5 +1,16 @@
# LoRa
`mpremote connect /dev/serial/by-id/usb-Silicon_Labs_CP2102_USB_to_UART_Bridge_Controller_0001-if00-port0 repl`
`mpremote connect /dev/serial/by-path/pci-0000:00:14.0-usb-0:6.1:1.0-port0 repl`
# MQTT
`mpremote connect /dev/serial/by-path/pci-0000:00:14.0-usb-0:6.2:1.0-port0 repl`
# UART
| LoRa | MQTT |
| --- | --- |
| 45 | P17 |
| 46 | P16 |
| GND | GND |
+1 -1
View File
@@ -61,7 +61,7 @@ while True:
command = uart_device.read()
print(f"[Main] Received command from WiFi Board: {command}")
uart_device.send(f"Hello from esp-32 lora ID {DEVICE_ID}")
# uart_device.send(f"Hello from esp-32 lora ID {DEVICE_ID}")
# 2. Send local metrics over the wire to the WiFi board every few seconds
# uart_device.send("Data Pack: LoRa Link RSSI -72dBm")
+3 -3
View File
@@ -1,6 +1,6 @@
# This file is executed on every boot (including wake-boot from deepsleep)
#import esp
#esp.osdebug(None)
import esp
esp.osdebug(True)
#import webrepl
#webrepl.start()
@@ -16,4 +16,4 @@ def do_connect(ssid, pwd):
print('network config:', sta_if.ifconfig())
# Attempt to connect to WiFi network
do_connect("Smartwave-1", 'Smartwave-prot-1')
# do_connect("Smartwave-1", 'Smartwave-prot-1')
+73 -19
View File
@@ -1,8 +1,11 @@
import _thread
import select
from machine import Pin
from shared import get_mqtt_client, get_uart, config
from sensors import temperature_gun
from shared import get_mqtt_client, get_uart, config, payloads
import time
import ujson as json
import sys
# Simple thread-safe queue list
msg_queue = []
@@ -13,17 +16,22 @@ def queue_publish(topic, payload):
with queue_lock:
msg_queue.append((topic, payload))
# --- Hardware & Client Setup ---
vext = Pin(19, Pin.OUT)
vext.value(0)
time.sleep_ms(100)
# --- INITIALIZE CAMERA ---
try:
# Pass your confirmed working SCL and SDA pins here
temperature_gun.init_camera(scl_pin=21, sda_pin=22, freq=100000)
except Exception as e:
print("[Main] Critical: Camera setup failed!")
sys.print_exception(e)
# --- READ DEVICE ID ---
try:
with open("device_id.txt", "r") as f:
DEVICE_ID = f.read().strip()
except Exception:
DEVICE_ID = "ESP32_Inconnu"
# --- MQTT SETUP ---
MQTT_CA_FILE = "/certs/ca.crt"
mqtt_client = get_mqtt_client(
@@ -34,8 +42,28 @@ mqtt_client = get_mqtt_client(
keepalive=config.MQTT_KEEPALIVE,
)
global orchestrator_id
orchestrator_id = None
def on_mqtt_message(message):
print("[MQTT Thread] Received message:", message)
# Try and parse the payload as json, but if it fails, just print the raw payload
payload_data=None
try:
payload_data = json.loads(message['payload'])
except Exception as e:
print("[MQTT Thread] Error parsing JSON:", e)
sys.print_exception(e)
pass # Maybe it's not JSON
if message['topic'] == config.MQTT_TOPIC_HELLO and payload_data and "id_orchestrator" in payload_data and payload_data["id_microwave"] == DEVICE_ID:
print("[MQTT Thread] Hello response received from orchestrator:", payload_data["id_orchestrator"])
global orchestrator_id
orchestrator_id = payload_data["id_orchestrator"]
# Unsubscribe from the hello topic since we got a response
mqtt_client.unsubscribe(config.MQTT_TOPIC_HELLO)
print("[MQTT Thread] Unsubscribed from topic:", config.MQTT_TOPIC_HELLO)
print("[MQTT Thread] Message processing complete.")
mqtt_client.set_callback(on_mqtt_message)
@@ -73,7 +101,7 @@ def mqtt_background_thread():
# 3. Handle Keepalive tracking manually
if time.time() - last_check >= 15:
print("[Thread] Sending keepalive ping...")
# print("[Thread] Sending keepalive ping...")
mqtt_client._client.ping()
last_check = time.time()
@@ -82,6 +110,7 @@ def mqtt_background_thread():
except Exception as e:
print("[Thread] Connection dropped or error encountered:", e)
sys.print_exception(e)
print("[Thread] Cleaning up socket context. Retrying in 5 seconds...")
# --- FIX FOR ERROR 23 (SOCKET LEAK) ---
@@ -99,29 +128,54 @@ def mqtt_background_thread():
except Exception:
pass
time.sleep(5)
# --- UART BACKGROUND THREAD ---
def uart_background_thread():
"""Background UART worker handling all serial operations safely."""
print("[Thread] Background UART worker started.")
uart_device = get_uart(uart_id=2, tx_pin=17, rx_pin=16)
while True:
try:
# 1. Check for incoming messages from the Heltec board
while uart_device.any():
incoming_msg = uart_device.read()
print(f"[Thread] Received from esp-lora over UART: {incoming_msg}")
# 2. Example: Send data to the Heltec board every 5 seconds
# uart_device.send("Status Check: WiFi Active")
# UART
uart_device = get_uart(uart_id=2, tx_pin=17, rx_pin=16)
time.sleep(5) # Fast responsive polling loop for local UART
except Exception as e:
print("[Thread] UART error encountered:", e)
time.sleep(5)
# --- Launch background worker ---
_thread.start_new_thread(mqtt_background_thread, ())
# _thread.start_new_thread(mqtt_background_thread, ())
# _thread.start_new_thread(uart_background_thread, ())
# --- MAIN APPLICATION THREAD (Core 0) ---
print("[Main] Main execution path active.")
time.sleep(2) # Give the thread a moment to initial connect
mqtt_hello_sent_timestamp = -config.MQTT_HELLO_INTERVAL
mqtt_client.subscribe(config.MQTT_TOPIC_HELLO, qos=config.MQTT_QOS)
while True:
print("[Main] Queueing a test message for MQTT...")
# Instead of direct publishing, push it to the queue safely
queue_publish(config.MQTT_TOPIC_SENSOR, "Hello from ESP32!")
# 1. Check if the Heltec V3 sent us something over the wire
while uart_device.any():
incoming_msg = uart_device.read()
print(f"[Main] Received from esp-lora over UART: {incoming_msg}")
# MQTT HELLO sent every x seconds until we get a response from the orchestrator
if (orchestrator_id == None and -(mqtt_hello_sent_timestamp - time.time()) > config.MQTT_HELLO_INTERVAL):
print("[Main] Attempting to send initial hello to orchestrator...")
queue_publish(config.MQTT_TOPIC_HELLO, payloads.mqtt_hello(DEVICE_ID))
mqtt_hello_sent_timestamp = time.time()
pass
# Sensors
print(f"[Main] Reading temperature from the gun sensor...")
temp = temperature_gun.read_temperature()
print(f"[Main] Temperature read: {temp}°C")
# 2. Example: Send data to the Heltec board every 5 seconds
# uart_device.send("Status Check: WiFi Active")
time.sleep_ms(200) # Fast responsive polling loop for local UART
time.sleep(1)
+1
View File
@@ -0,0 +1 @@
import sensors.temperature_gun as temperature_gun
@@ -0,0 +1,969 @@
"""
Temperatue gun sensor module
using the MLX90640-D55/D110 sensor. This module provides a function to read the temperature from the gun sensor.
Resolution of 32x24 pixels,
I2C interface
Noise Equivalent Temperature difference (NETD) is 0.1K RMS @ 1Hz refresh rate
"""
import machine # type: ignore
import math
import struct
import time
from micropython import const# Some libraries that we will use
import time
class RefreshRate: # pylint: disable=too-few-public-methods
""" Enum-like class for MLX90640's refresh rate """
REFRESH_0_5_HZ = const(0b000) # 0.5Hz
REFRESH_1_HZ = const(0b001) # 1Hz
REFRESH_2_HZ = const(0b010) # 2Hz
REFRESH_4_HZ = const(0b011) # 4Hz
REFRESH_8_HZ = const(0b100) # 8Hz
REFRESH_16_HZ = const(0b101) # 16Hz
REFRESH_32_HZ = const(0b110) # 32Hz
REFRESH_64_HZ = const(0b111) # 64Hz
class ContextManaged:
"""An object that automatically deinitializes hardware with a context manager."""
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.deinit()
# pylint: disable=no-self-use
def deinit(self):
"""Free any hardware used by the object."""
return
class Lockable(ContextManaged):
"""An object that must be locked to prevent collisions on a microcontroller resource."""
_locked = False
def try_lock(self):
"""Attempt to grab the lock. Return True on success, False if the lock is already taken."""
if self._locked:
return False
self._locked = True
return True
def unlock(self):
"""Release the lock so others may use the resource."""
if self._locked:
self._locked = False
else:
raise ValueError("Not locked")
class I2C(Lockable):
def __init__(self, pins=(21, 22), frequency=100000):
self.init(pins, frequency)
def init(self, pins, frequency):
self.deinit()
# 1. Force the ESP32 to activate its internal pull-up resistors on these pins
self._pins = (
machine.Pin(int(pins[0]), machine.Pin.IN, machine.Pin.PULL_UP),
machine.Pin(int(pins[1]), machine.Pin.IN, machine.Pin.PULL_UP)
)
try:
# 2. Bypasses the glitchy ESP32 hardware block using SoftI2C
# (Note: SoftI2C does not take a bus ID number like '0')
self._i2c = machine.SoftI2C(scl=self._pins[0], sda=self._pins[1], freq=frequency)
except RuntimeError:
raise
print(f"Created resilient SoftI2C: {self._i2c}")
def deinit(self):
try:
del self._i2c
except AttributeError:
pass
def scan(self):
return self._i2c.scan()
def readfrom_into(self, address, buffer, *, start=0, end=None):
if start is not 0 or end is not None:
if end is None:
end = len(buffer)
buffer = memoryview(buffer)[start:end]
stop = True # remove for efficiency later
return self._i2c.readfrom_into(address, buffer)
def writeto(self, address, buffer, *, start=0, end=None, stop=True):
if isinstance(buffer, str):
buffer = bytes([ord(x) for x in buffer])
if start is not 0 or end is not None:
if end is None:
return self._i2c.writeto(address, memoryview(buffer)[start:], stop)
else:
return self._i2c.writeto(address, memoryview(buffer)[start:end], stop)
return self._i2c.writeto(address, buffer, stop)
class I2CDevice:
def __init__(self, i2c, device_address, probe=True):
self.i2c = i2c
self._has_write_read = False # hasattr(self.i2c, "writeto_then_readfrom") --> has been turned to False
self.device_address = device_address
if probe:
self.__probe_for_device()
def readinto(self, buf, *, start=0, end=None):
if end is None:
end = len(buf)
self.i2c.readfrom_into(self.device_address, buf, start=start, end=end)
def write(self, buf, *, start=0, end=None, stop=True):
if end is None:
end = len(buf)
self.i2c.writeto(self.device_address, buf, start=start, end=end, stop=stop)
# pylint: disable-msg=too-many-arguments
def write_then_readinto(
self,
out_buffer,
in_buffer,
*,
out_start=0,
out_end=None,
in_start=0,
in_end=None,
stop=False
):
if out_end is None:
out_end = len(out_buffer)
if in_end is None:
in_end = len(in_buffer)
if stop:
raise ValueError("Stop must be False. Use writeto instead.")
if self._has_write_read:
#print("c",dir(self.i2c))
# In linux, at least, this is a special kernel function call
self.i2c.writeto_then_readfrom(
self.device_address,
out_buffer,
in_buffer,
out_start=out_start,
out_end=out_end,
in_start=in_start,
in_end=in_end,
)
else:
# If we don't have a special implementation, we can fake it with two calls
self.i2c.writeto(self.device_address, out_buffer, stop=False) # These lines have been changed to make it work with wipy micropython I2C module
#self.write(out_buffer, start=out_start, end=out_end, stop=False)
#self.readinto(in_buffer, start=in_start, end=in_end)
self.i2c.readfrom_into(self.device_address, in_buffer) # These lines have been changed to make it work with wipy micropython I2C module
# pylint: enable-msg=too-many-arguments
def __enter__(self):
while not self.i2c.try_lock():
pass
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.i2c.unlock()
return False
def __probe_for_device(self):
"""
Try to read a byte from an address,
if you get an OSError it means the device is not there
or that the device does not support these means of probing
"""
while not self.i2c.try_lock():
pass
try:
self.i2c.writeto(self.device_address, b"")
except OSError:
# some OS's dont like writing an empty bytesting...
# Retry by reading a byte
try:
result = bytearray(1)
self.i2c.readfrom_into(self.device_address, result)
except OSError:
raise ValueError("No I2C device at address: %x" % self.device_address)
finally:
self.i2c.unlock()
eeData = [0] * const(832)
I2C_READ_LEN = const(2048)
SCALEALPHA = const(0.000001)
MLX90640_DEVICEID1 = const(0x2407)
OPENAIR_TA_SHIFT = const(8)
class MLX90640: # pylint: disable=too-many-instance-attributes
"""Interface to the MLX90640 temperature sensor."""
kVdd = 0
vdd25 = 0
KvPTAT = 0
KtPTAT = 0
vPTAT25 = 0
alphaPTAT = 0
gainEE = 0
tgc = 0
KsTa = 0
resolutionEE = 0
calibrationModeEE = 0
ksTo = [0] * 5
ct = [0] * 5
alpha = [0] * 768
alphaScale = 0
offset = [0] * 768
kta = [0] * 768
ktaScale = 0
kv = [0] * 768
kvScale = 0
cpAlpha = [0] * 2
cpOffset = [0] * 2
ilChessC = [0] * 3
brokenPixels = [0xFFFF] * 5
outlierPixels = [0xFFFF] * 5
cpKta = 0
cpKv = 0
def __init__(self, i2c_bus, address=0x33):
self.i2c_device = I2CDevice(i2c_bus, address)
self._I2CReadWords(0x2400, eeData)
# print(eeData)
self._ExtractParameters()
@property
def serial_number(self):
""" 3-item tuple of hex values that are unique to each MLX90640 """
serialWords = [0, 0, 0]
self._I2CReadWords(MLX90640_DEVICEID1, serialWords)
return serialWords
@property
def refresh_rate(self):
""" How fast the MLX90640 will spit out data. Start at lowest speed in
RefreshRate and then slowly increase I2C clock rate and rate until you
max out. The sensor does not like it if the I2C host cannot 'keep up'!"""
controlRegister = [0]
self._I2CReadWords(0x800D, controlRegister)
return (controlRegister[0] >> 7) & 0x07
@refresh_rate.setter
def refresh_rate(self, rate):
controlRegister = [0]
value = (rate & 0x7) << 7
self._I2CReadWords(0x800D, controlRegister)
value |= controlRegister[0] & 0xFC7F
self._I2CWriteWord(0x800D, value)
def getFrame(self, framebuf):
""" Request both 'halves' of a frame from the sensor, merge them
and calculate the temperature in C for each of 32x24 pixels. Placed
into the 768-element array passed in! """
emissivity = 0.95
tr = 23.15
mlx90640Frame = [0] * 834
for _ in range(2):
status = self._GetFrameData(mlx90640Frame)
if status < 0:
raise RuntimeError("Frame data error")
# For a MLX90640 in the open air the shift is -8 degC.
tr = self._GetTa(mlx90640Frame) - OPENAIR_TA_SHIFT
self._CalculateTo(mlx90640Frame, emissivity, tr, framebuf)
def _GetFrameData(self, frameData):
dataReady = 0
cnt = 0
statusRegister = [0]
controlRegister = [0]
while dataReady == 0:
self._I2CReadWords(0x8000, statusRegister)
dataReady = statusRegister[0] & 0x0008
# print("ready status: 0x%x" % dataReady)
while (dataReady != 0) and (cnt < 5):
self._I2CWriteWord(0x8000, 0x0030)
# print("Read frame", cnt)
self._I2CReadWords(0x0400, frameData, end=832)
self._I2CReadWords(0x8000, statusRegister)
dataReady = statusRegister[0] & 0x0008
# print("frame ready: 0x%x" % dataReady)
cnt += 1
if cnt > 4:
raise RuntimeError("Too many retries")
self._I2CReadWords(0x800D, controlRegister)
frameData[832] = controlRegister[0]
frameData[833] = statusRegister[0] & 0x0001
return frameData[833]
def _GetTa(self, frameData):
vdd = self._GetVdd(frameData)
ptat = frameData[800]
if ptat > 32767:
ptat -= 65536
ptatArt = frameData[768]
if ptatArt > 32767:
ptatArt -= 65536
ptatArt = (ptat / (ptat * self.alphaPTAT + ptatArt)) * math.pow(2, 18)
ta = ptatArt / (1 + self.KvPTAT * (vdd - 3.3)) - self.vPTAT25
ta = ta / self.KtPTAT + 25
return ta
def _GetVdd(self, frameData):
vdd = frameData[810]
if vdd > 32767:
vdd -= 65536
resolutionRAM = (frameData[832] & 0x0C00) >> 10
resolutionCorrection = math.pow(2, self.resolutionEE) / math.pow(
2, resolutionRAM
)
vdd = (resolutionCorrection * vdd - self.vdd25) / self.kVdd + 3.3
return vdd
def _CalculateTo(self, frameData, emissivity, tr, result):
# pylint: disable=too-many-locals, too-many-branches, too-many-statements
subPage = frameData[833]
alphaCorrR = [0] * 4
irDataCP = [0, 0]
vdd = self._GetVdd(frameData)
ta = self._GetTa(frameData)
ta4 = ta + 273.15
ta4 = ta4 * ta4
ta4 = ta4 * ta4
tr4 = tr + 273.15
tr4 = tr4 * tr4
tr4 = tr4 * tr4
taTr = tr4 - (tr4 - ta4) / emissivity
ktaScale = math.pow(2, self.ktaScale)
kvScale = math.pow(2, self.kvScale)
alphaScale = math.pow(2, self.alphaScale)
alphaCorrR[0] = 1 / (1 + self.ksTo[0] * 40)
alphaCorrR[1] = 1
alphaCorrR[2] = 1 + self.ksTo[1] * self.ct[2]
alphaCorrR[3] = alphaCorrR[2] * (1 + self.ksTo[2] * (self.ct[3] - self.ct[2]))
# --------- Gain calculation -----------------------------------
gain = frameData[778]
if gain > 32767:
gain -= 65536
gain = self.gainEE / gain
# --------- To calculation -------------------------------------
mode = (frameData[832] & 0x1000) >> 5
irDataCP[0] = frameData[776]
irDataCP[1] = frameData[808]
for i in range(2):
if irDataCP[i] > 32767:
irDataCP[i] -= 65536
irDataCP[i] *= gain
irDataCP[0] -= (
self.cpOffset[0]
* (1 + self.cpKta * (ta - 25))
* (1 + self.cpKv * (vdd - 3.3))
)
if mode == self.calibrationModeEE:
irDataCP[1] -= (
self.cpOffset[1]
* (1 + self.cpKta * (ta - 25))
* (1 + self.cpKv * (vdd - 3.3))
)
else:
irDataCP[1] -= (
(self.cpOffset[1] + self.ilChessC[0])
* (1 + self.cpKta * (ta - 25))
* (1 + self.cpKv * (vdd - 3.3))
)
for pixelNumber in range(768):
ilPattern = pixelNumber // 32 - (pixelNumber // 64) * 2
chessPattern = ilPattern ^ (pixelNumber - (pixelNumber // 2) * 2)
conversionPattern = (
(pixelNumber + 2) // 4
- (pixelNumber + 3) // 4
+ (pixelNumber + 1) // 4
- pixelNumber // 4
) * (1 - 2 * ilPattern)
if mode == 0:
pattern = ilPattern
else:
pattern = chessPattern
if pattern == frameData[833]:
irData = frameData[pixelNumber]
if irData > 32767:
irData -= 65536
irData *= gain
kta = self.kta[pixelNumber] / ktaScale
kv = self.kv[pixelNumber] / kvScale
irData -= (
self.offset[pixelNumber]
* (1 + kta * (ta - 25))
* (1 + kv * (vdd - 3.3))
)
if mode != self.calibrationModeEE:
irData += (
self.ilChessC[2] * (2 * ilPattern - 1)
- self.ilChessC[1] * conversionPattern
)
irData = irData - self.tgc * irDataCP[subPage]
irData /= emissivity
alphaCompensated = SCALEALPHA * alphaScale / self.alpha[pixelNumber]
alphaCompensated *= 1 + self.KsTa * (ta - 25)
Sx = (
alphaCompensated
* alphaCompensated
* alphaCompensated
* (irData + alphaCompensated * taTr)
)
Sx = math.sqrt(math.sqrt(Sx)) * self.ksTo[1]
To = (
math.sqrt(
math.sqrt(
irData
/ (alphaCompensated * (1 - self.ksTo[1] * 273.15) + Sx)
+ taTr
)
)
- 273.15
)
if To < self.ct[1]:
torange = 0
elif To < self.ct[2]:
torange = 1
elif To < self.ct[3]:
torange = 2
else:
torange = 3
To = (
math.sqrt(
math.sqrt(
irData
/ (
alphaCompensated
* alphaCorrR[torange]
* (1 + self.ksTo[torange] * (To - self.ct[torange]))
)
+ taTr
)
)
- 273.15
)
result[pixelNumber] = To
# pylint: enable=too-many-locals, too-many-branches, too-many-statements
def _ExtractParameters(self):
self._ExtractVDDParameters()
self._ExtractPTATParameters()
self._ExtractGainParameters()
self._ExtractTgcParameters()
self._ExtractResolutionParameters()
self._ExtractKsTaParameters()
self._ExtractKsToParameters()
self._ExtractCPParameters()
self._ExtractAlphaParameters()
self._ExtractOffsetParameters()
self._ExtractKtaPixelParameters()
self._ExtractKvPixelParameters()
self._ExtractCILCParameters()
self._ExtractDeviatingPixels()
def _ExtractVDDParameters(self):
# extract VDD
self.kVdd = (eeData[51] & 0xFF00) >> 8
if self.kVdd > 127:
self.kVdd -= 256 # convert to signed
self.kVdd *= 32
self.vdd25 = eeData[51] & 0x00FF
self.vdd25 = ((self.vdd25 - 256) << 5) - 8192
def _ExtractPTATParameters(self):
# extract PTAT
self.KvPTAT = (eeData[50] & 0xFC00) >> 10
if self.KvPTAT > 31:
self.KvPTAT -= 64
self.KvPTAT /= 4096
self.KtPTAT = eeData[50] & 0x03FF
if self.KtPTAT > 511:
self.KtPTAT -= 1024
self.KtPTAT /= 8
self.vPTAT25 = eeData[49]
self.alphaPTAT = (eeData[16] & 0xF000) / math.pow(2, 14) + 8
def _ExtractGainParameters(self):
# extract Gain
self.gainEE = eeData[48]
if self.gainEE > 32767:
self.gainEE -= 65536
def _ExtractTgcParameters(self):
# extract Tgc
#print(eeData[60])
self.tgc = eeData[60] & 0x00FF
#print(self.tgc)
if self.tgc > 127:
self.tgc -= 256
self.tgc /= 32
#print(self.tgc)
def _ExtractResolutionParameters(self):
# extract resolution
self.resolutionEE = (eeData[56] & 0x3000) >> 12
def _ExtractKsTaParameters(self):
# extract KsTa
self.KsTa = (eeData[60] & 0xFF00) >> 8
if self.KsTa > 127:
self.KsTa -= 256
self.KsTa /= 8192
def _ExtractKsToParameters(self):
# extract ksTo
step = ((eeData[63] & 0x3000) >> 12) * 10
self.ct[0] = -40
self.ct[1] = 0
self.ct[2] = (eeData[63] & 0x00F0) >> 4
self.ct[3] = (eeData[63] & 0x0F00) >> 8
self.ct[2] *= step
self.ct[3] = self.ct[2] + self.ct[3] * step
KsToScale = (eeData[63] & 0x000F) + 8
KsToScale = 1 << KsToScale
self.ksTo[0] = eeData[61] & 0x00FF
self.ksTo[1] = (eeData[61] & 0xFF00) >> 8
self.ksTo[2] = eeData[62] & 0x00FF
self.ksTo[3] = (eeData[62] & 0xFF00) >> 8
for i in range(4):
if self.ksTo[i] > 127:
self.ksTo[i] -= 256
self.ksTo[i] /= KsToScale
self.ksTo[4] = -0.0002
def _ExtractCPParameters(self):
# extract CP
offsetSP = [0] * 2
alphaSP = [0] * 2
alphaScale = ((eeData[32] & 0xF000) >> 12) + 27
offsetSP[0] = eeData[58] & 0x03FF
if offsetSP[0] > 511:
offsetSP[0] -= 1024
offsetSP[1] = (eeData[58] & 0xFC00) >> 10
if offsetSP[1] > 31:
offsetSP[1] -= 64
offsetSP[1] += offsetSP[0]
alphaSP[0] = eeData[57] & 0x03FF
if alphaSP[0] > 511:
alphaSP[0] -= 1024
alphaSP[0] /= math.pow(2, alphaScale)
alphaSP[1] = (eeData[57] & 0xFC00) >> 10
if alphaSP[1] > 31:
alphaSP[1] -= 64
alphaSP[1] = (1 + alphaSP[1] / 128) * alphaSP[0]
cpKta = eeData[59] & 0x00FF
if cpKta > 127:
cpKta -= 256
ktaScale1 = ((eeData[56] & 0x00F0) >> 4) + 8
self.cpKta = cpKta / math.pow(2, ktaScale1)
cpKv = (eeData[59] & 0xFF00) >> 8
if cpKv > 127:
cpKv -= 256
kvScale = (eeData[56] & 0x0F00) >> 8
self.cpKv = cpKv / math.pow(2, kvScale)
self.cpAlpha[0] = alphaSP[0]
self.cpAlpha[1] = alphaSP[1]
self.cpOffset[0] = offsetSP[0]
self.cpOffset[1] = offsetSP[1]
#print(self.cpAlpha[0])
#print(self.cpAlpha[1])
def _ExtractAlphaParameters(self):
# extract alpha
accRemScale = eeData[32] & 0x000F
accColumnScale = (eeData[32] & 0x00F0) >> 4
accRowScale = (eeData[32] & 0x0F00) >> 8
alphaScale = ((eeData[32] & 0xF000) >> 12) + 30
alphaRef = eeData[33]
accRow = [0] * 24
accColumn = [0] * 32
alphaTemp = [0] * 768
for i in range(6):
p = i * 4
accRow[p + 0] = eeData[34 + i] & 0x000F
accRow[p + 1] = (eeData[34 + i] & 0x00F0) >> 4
accRow[p + 2] = (eeData[34 + i] & 0x0F00) >> 8
accRow[p + 3] = (eeData[34 + i] & 0xF000) >> 12
for i in range(24):
if accRow[i] > 7:
accRow[i] -= 16
for i in range(8):
p = i * 4
accColumn[p + 0] = eeData[40 + i] & 0x000F
accColumn[p + 1] = (eeData[40 + i] & 0x00F0) >> 4
accColumn[p + 2] = (eeData[40 + i] & 0x0F00) >> 8
accColumn[p + 3] = (eeData[40 + i] & 0xF000) >> 12
for i in range(32):
if accColumn[i] > 7:
accColumn[i] -= 16
for i in range(24):
for j in range(32):
p = 32 * i + j
alphaTemp[p] = (eeData[64 + p] & 0x03F0) >> 4
if alphaTemp[p] > 31:
alphaTemp[p] -= 64
alphaTemp[p] *= 1 << accRemScale
alphaTemp[p] += (
alphaRef
+ (accRow[i] << accRowScale)
+ (accColumn[j] << accColumnScale)
)
alphaTemp[p] /= math.pow(2, alphaScale)
alphaTemp[p] -= self.tgc * (self.cpAlpha[0] + self.cpAlpha[1]) / 2
alphaTemp[p] = SCALEALPHA / alphaTemp[p]
# print("alphaTemp: ", alphaTemp)
temp = max(alphaTemp)
#print("temp", temp)
alphaScale = 0
while temp < 32768:
temp *= 2
alphaScale += 1
for i in range(768):
temp = alphaTemp[i] * math.pow(2, alphaScale)
self.alpha[i] = int(temp + 0.5)
self.alphaScale = alphaScale
def _ExtractOffsetParameters(self):
# extract offset
occRow = [0] * 24
occColumn = [0] * 32
occRemScale = eeData[16] & 0x000F
occColumnScale = (eeData[16] & 0x00F0) >> 4
occRowScale = (eeData[16] & 0x0F00) >> 8
offsetRef = eeData[17]
if offsetRef > 32767:
offsetRef -= 65536
for i in range(6):
p = i * 4
occRow[p + 0] = eeData[18 + i] & 0x000F
occRow[p + 1] = (eeData[18 + i] & 0x00F0) >> 4
occRow[p + 2] = (eeData[18 + i] & 0x0F00) >> 8
occRow[p + 3] = (eeData[18 + i] & 0xF000) >> 12
for i in range(24):
if occRow[i] > 7:
occRow[i] -= 16
for i in range(8):
p = i * 4
occColumn[p + 0] = eeData[24 + i] & 0x000F
occColumn[p + 1] = (eeData[24 + i] & 0x00F0) >> 4
occColumn[p + 2] = (eeData[24 + i] & 0x0F00) >> 8
occColumn[p + 3] = (eeData[24 + i] & 0xF000) >> 12
for i in range(32):
if occColumn[i] > 7:
occColumn[i] -= 16
for i in range(24):
for j in range(32):
p = 32 * i + j
self.offset[p] = (eeData[64 + p] & 0xFC00) >> 10
if self.offset[p] > 31:
self.offset[p] -= 64
self.offset[p] *= 1 << occRemScale
self.offset[p] += (
offsetRef
+ (occRow[i] << occRowScale)
+ (occColumn[j] << occColumnScale)
)
def _ExtractKtaPixelParameters(self): # pylint: disable=too-many-locals
# extract KtaPixel
KtaRC = [0] * 4
ktaTemp = [0] * 768
KtaRoCo = (eeData[54] & 0xFF00) >> 8
if KtaRoCo > 127:
KtaRoCo -= 256
KtaRC[0] = KtaRoCo
KtaReCo = eeData[54] & 0x00FF
if KtaReCo > 127:
KtaReCo -= 256
KtaRC[2] = KtaReCo
KtaRoCe = (eeData[55] & 0xFF00) >> 8
if KtaRoCe > 127:
KtaRoCe -= 256
KtaRC[1] = KtaRoCe
KtaReCe = eeData[55] & 0x00FF
if KtaReCe > 127:
KtaReCe -= 256
KtaRC[3] = KtaReCe
ktaScale1 = ((eeData[56] & 0x00F0) >> 4) + 8
ktaScale2 = eeData[56] & 0x000F
for i in range(24):
for j in range(32):
p = 32 * i + j
split = 2 * (p // 32 - (p // 64) * 2) + p % 2
ktaTemp[p] = (eeData[64 + p] & 0x000E) >> 1
if ktaTemp[p] > 3:
ktaTemp[p] -= 8
ktaTemp[p] *= 1 << ktaScale2
ktaTemp[p] += KtaRC[split]
ktaTemp[p] /= math.pow(2, ktaScale1)
# ktaTemp[p] = ktaTemp[p] * mlx90640->offset[p];
temp = abs(ktaTemp[0])
for kta in ktaTemp:
temp = max(temp, abs(kta))
ktaScale1 = 0
while temp < 64:
temp *= 2
ktaScale1 += 1
for i in range(768):
temp = ktaTemp[i] * math.pow(2, ktaScale1)
if temp < 0:
self.kta[i] = int(temp - 0.5)
else:
self.kta[i] = int(temp + 0.5)
self.ktaScale = ktaScale1
def _ExtractKvPixelParameters(self):
KvT = [0] * 4
kvTemp = [0] * 768
KvRoCo = (eeData[52] & 0xF000) >> 12
if KvRoCo > 7:
KvRoCo -= 16
KvT[0] = KvRoCo
KvReCo = (eeData[52] & 0x0F00) >> 8
if KvReCo > 7:
KvReCo -= 16
KvT[2] = KvReCo
KvRoCe = (eeData[52] & 0x00F0) >> 4
if KvRoCe > 7:
KvRoCe -= 16
KvT[1] = KvRoCe
KvReCe = eeData[52] & 0x000F
if KvReCe > 7:
KvReCe -= 16
KvT[3] = KvReCe
kvScale = (eeData[56] & 0x0F00) >> 8
for i in range(24):
for j in range(32):
p = 32 * i + j
split = 2 * (p // 32 - (p // 64) * 2) + p % 2
kvTemp[p] = KvT[split]
kvTemp[p] /= math.pow(2, kvScale)
# kvTemp[p] = kvTemp[p] * mlx90640->offset[p];
temp = abs(kvTemp[0])
for kv in kvTemp:
temp = max(temp, abs(kv))
kvScale = 0
while temp < 64:
temp *= 2
kvScale += 1
for i in range(768):
temp = kvTemp[i] * math.pow(2, kvScale)
if temp < 0:
self.kv[i] = int(temp - 0.5)
else:
self.kv[i] = int(temp + 0.5)
self.kvScale = kvScale
def _ExtractCILCParameters(self):
ilChessC = [0] * 3
self.calibrationModeEE = (eeData[10] & 0x0800) >> 4
self.calibrationModeEE = self.calibrationModeEE ^ 0x80
ilChessC[0] = eeData[53] & 0x003F
if ilChessC[0] > 31:
ilChessC[0] -= 64
ilChessC[0] /= 16.0
ilChessC[1] = (eeData[53] & 0x07C0) >> 6
if ilChessC[1] > 15:
ilChessC[1] -= 32
ilChessC[1] /= 2.0
ilChessC[2] = (eeData[53] & 0xF800) >> 11
if ilChessC[2] > 15:
ilChessC[2] -= 32
ilChessC[2] /= 8.0
self.ilChessC = ilChessC
def _ExtractDeviatingPixels(self):
self.brokenPixels = [0xFFFF] * 5
self.outlierPixels = [0xFFFF] * 5
pixCnt = 0
brokenPixCnt = 0
outlierPixCnt = 0
while (pixCnt < 768) and (brokenPixCnt < 5) and (outlierPixCnt < 5):
if eeData[pixCnt + 64] == 0:
self.brokenPixels[brokenPixCnt] = pixCnt
brokenPixCnt += 1
elif (eeData[pixCnt + 64] & 0x0001) != 0:
self.outlierPixels[outlierPixCnt] = pixCnt
outlierPixCnt += 1
pixCnt += 1
if brokenPixCnt > 4:
raise RuntimeError("More than 4 broken pixels")
if outlierPixCnt > 4:
raise RuntimeError("More than 4 outlier pixels")
if (brokenPixCnt + outlierPixCnt) > 4:
raise RuntimeError("More than 4 faulty pixels")
# print("Found %d broken pixels, %d outliers" % (brokenPixCnt, outlierPixCnt))
# TODO INCOMPLETE
def _I2CWriteWord(self, writeAddress, data):
cmd = bytearray(4)
cmd[0] = writeAddress >> 8
cmd[1] = writeAddress & 0x00FF
cmd[2] = data >> 8
cmd[3] = data & 0x00FF
dataCheck = [0]
with self.i2c_device as i2c:
i2c.write(cmd)
# print("Wrote:", [hex(i) for i in cmd])
time.sleep(0.001)
self._I2CReadWords(writeAddress, dataCheck)
# print("dataCheck: 0x%x" % dataCheck[0])
# if (dataCheck != data):
# return -2
_inbuf = bytearray(2 * I2C_READ_LEN)
def _I2CReadWords(self, addr, buffer, *, end=None):
# stamp = time.monotonic()
if end is None:
remainingWords = len(buffer)
else:
remainingWords = end
offset = 0
addrbuf = bytearray(2)
# inbuf = bytearray(2 * I2C_READ_LEN)
inbuf = self._inbuf
with self.i2c_device as i2c:
while remainingWords:
addrbuf[0] = addr >> 8 # MSB
addrbuf[1] = addr & 0xFF # LSB
read_words = min(remainingWords, I2C_READ_LEN)
i2c.write_then_readinto(
addrbuf, inbuf, in_end=read_words * 2
) # in bytes
# print("-> ", [hex(i) for i in addrbuf])
outwords = struct.unpack(
">" + "H" * read_words, inbuf[0 : read_words * 2]
)
# print("<- (", read_words, ")", [hex(i) for i in outwords])
for i, w in enumerate(outwords):
buffer[offset + i] = w
offset += read_words
remainingWords -= read_words
addr += read_words
ixc = None
mlx = None
frame = None
def init_camera(scl_pin=22, sda_pin=21, freq=100000):
"""Explicitly initializes the I2C bus and camera after power is stable."""
global ixc, mlx, frame
print(f"[Camera] Initializing I2C on SCL:{scl_pin}, SDA:{sda_pin} at {freq}Hz...")
ixc = I2C(pins=(scl_pin, sda_pin), frequency=freq)
print("[Camera] Probing for MLX90640...")
mlx = MLX90640(ixc)
# Bonus: Your wiki snapshot recommends 16Hz for smooth images!
mlx.refresh_rate = RefreshRate.REFRESH_16_HZ
frame = [0] * 768
print("[Camera] Setup successful!")
def read_temperature():
if mlx is None:
print("[Camera] Error: Camera not initialized. Call init_camera() first.")
return None
try:
print("Querying camera...")
mlx.getFrame(frame)
return frame
except Exception as e:
print(f"[Camera] Read error: {e}")
return None
+2 -2
View File
@@ -5,7 +5,7 @@ SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
REPO_ROOT=$(dirname -- "$SCRIPT_DIR")
PYTHON_BIN="${PYTHON_BIN:-python3}"
PYTHON_SCRIPT="${1:-$SCRIPT_DIR/main.py}"
REQUIREMENTS_FILE="$REPO_ROOT/requirements.txt"
REQUIREMENTS_FILE="$SCRIPT_DIR/requirements.txt"
if [ ! -f "$PYTHON_SCRIPT" ]; then
echo "Python script not found: $PYTHON_SCRIPT" >&2
@@ -19,7 +19,7 @@ cd "$SCRIPT_DIR"
# 2. On lance Docker en arrière-plan
echo "Démarrage des conteneurs Docker..."
docker compose pull
# docker compose pull
docker compose up -d --remove-orphans
# 3. Installation des dépendances (sans '--user' si on est déjà root sous systemd)
+101 -24
View File
@@ -1,7 +1,12 @@
import json
import threading
import queue
import time
from shared import get_lora, get_mqtt_client, deviceTypes, config
import traceback
from orchestrateur.sensors import gps
from shared import get_lora, get_mqtt_client, deviceTypes, config, payloads
from shared.logging import log
from sensors import ultrasonicRanger, temp_hum, button, camera
# --- Read Unique Device ID ---
try:
@@ -49,6 +54,7 @@ mqtt_client = get_mqtt_client(
)
mqtt_client.connect()
mqtt_client.subscribe(config.MQTT_TOPIC_SENSOR, qos=config.MQTT_QOS)
mqtt_client.subscribe(config.MQTT_TOPIC_HELLO, qos=config.MQTT_QOS)
print(f"Subscribed to topic: {config.MQTT_TOPIC_SENSOR}")
# --- THE CRUCIAL PAHO FIX ---
@@ -66,51 +72,122 @@ def mqtt_listener():
message = mqtt_client.get_message()
if message:
# Try to parse the payload as a python dictionary, but if it fails, just print the raw payload
try:
payload = json.loads(message['payload'])
except Exception as e:
print(f"Error parsing MQTT payload: {e}")
payload = message['payload'] # Fallback to raw payload if parsing fails
print(f"\n[Thread MQTT] Message reçu : {message}")
data_queue.put({"source": "MQTT", "data": message})
data_queue.put({"source": "MQTT", "topic": message['topic'] ,"data": payload})
# --- THE CPU FIX ---
# Sleep for 100ms. Prevents the thread from turning into an infinite 100% CPU hog.
time.sleep(0.1)
time.sleep(0.2)
# Button
button_state = False
def button_callback():
global button_state
button_state = not button_state
print(f"\n[Thread Button] Button state changed to: {button_state}")
button.set_callback(button_callback)
# Launch background monitoring workers
threading.Thread(target=lora_listener, daemon=True).start()
threading.Thread(target=mqtt_listener, daemon=True).start()
# threading.Thread(target=lora_listener, daemon=True).start()
# threading.Thread(target=mqtt_listener, daemon=True).start()
# Launch button monitoring thread
button.start_button_monitoring_thread()
print("Orchestrateur prêt. Le main loop est libre.")
# Sensor reading
def read_sensors():
"""Read all sensors and return a dictionary of their values."""
log("\nLecture des capteurs...")
sensor_data = {}
# Read Ultrasonic Ranger
distance = ultrasonicRanger.get_dish_height()
if distance is not None:
log(f"\nLecture du capteur Ultrason : {distance}")
sensor_data["ultrasonic_distance"] = distance
# Read Temperature and Humidity
temperature, humidity = temp_hum.get_temperature_and_humidity()
if temperature is not None and humidity is not None:
log(f"\nLecture du capteur Temp/Hum : {temperature}, {humidity}")
sensor_data["temperature"] = temperature
sensor_data["humidity"] = humidity
# Read GPS Data
gps_data = gps.get_gps_data()
if gps_data:
log(f"\nLecture du capteur GPS : {gps_data}")
sensor_data["gps"] = gps_data
# Camera
picture_bytes = None
try:
picture_bytes = camera.get_picture()
log(f"\nLecture du capteur Caméra : {len(picture_bytes)} bytes")
sensor_data["camera_image"] = picture_bytes
except Exception as e:
log(f"Error reading camera data: {e}")
# Read Button State (last because he can still change state while reading other sensors)
sensor_data["button_state"] = button_state
return sensor_data
# --- MAIN EXECUTION LOOP ---
while True:
try:
# Check for non-heartbeat data safely
# Check for non-heartbeat data
try:
msg = data_queue.get(block=False)
print(f"\n[Main Loop] Données traitées : {msg['data']}")
# print(msg)
if msg["source"] == "LoRa":
print(f"\n[Main Loop] LoRa : Données traitées : {msg['data']}")
elif msg["source"] == "MQTT":
if (msg["topic"] == config.MQTT_TOPIC_HELLO.decode('utf-8')):
if ("id_orchestrator" in msg["data"] and msg["data"]["id_orchestrator"] == DEVICE_ID):
# Do not answer to messages coming from me
continue
microwave_id = msg["data"]["id_microwave"]
print(f"\n[Main Loop] MQTT : Hello reçu de {microwave_id}.")
# Responds
mqtt_client.publish(config.MQTT_TOPIC_HELLO, payloads.mqtt_hello_ack(DEVICE_ID, microwave_id), qos=config.MQTT_QOS)
print(f"[Main Loop] MQTT : Réponse Hello envoyée à {microwave_id}.")
# TODO : Save in database
print(f"\n[Main Loop] MQTT : Données traitées : {msg['data']}")
except queue.Empty:
pass
# DEBUG : Read sensors
sensor_values = read_sensors()
if sensor_values:
sensor_values_print = sensor_values.copy()
if "camera_image" in sensor_values_print:
sensor_values_print["camera_image"] = f"<{len(sensor_values_print['camera_image'])} bytes>"
print(f"\nCapteurs Données lues : {sensor_values_print}")
time.sleep(1)
# Publish debug telemetry message
print("[Main Loop] Envoi d'un message de debug sur MQTT...")
response = mqtt_client.publish(
config.MQTT_TOPIC_COOKING,
f"Orchestrateur actif, ID: {DEVICE_ID}",
qos=config.MQTT_QOS
)
# This will now unblock instantly because loop_start() handles the delivery confirmation!
response.wait_for_publish()
print("[Main Loop] Message de debug publié avec succès.")
time.sleep(9)
time.sleep(3)
except KeyboardInterrupt:
break
except Exception as e:
traceback.print_exc()
time.sleep(1) # Prevents rapid error logging in case of persistent issues
# Clean termination
if hasattr(mqtt_client._client, "loop_stop"):
mqtt_client._client.loop_stop()
mqtt_client.close()
mqtt_client.close()
+6
View File
@@ -0,0 +1,6 @@
paho-mqtt>=1.6,<3
pyserial>=3.5,<4
# picamera2>=0.3.36,<4 # → Installed with apt install python3-picamera2
# OpenCV
# sudo apt install -y python3-opencv
# sudo apt install -y opencv-data
+7
View File
@@ -0,0 +1,7 @@
# import grovepi
import sensors.ultrasonicRanger as ultrasonicRanger
import sensors.temp_hum as temp_hum
import sensors.button as button
import sensors.gps as gps
import sensors.camera as camera
+44
View File
@@ -0,0 +1,44 @@
import grovepi
import time
import threading
from sensors.lock import grove_lock
from shared.logging import log
button = 2
button_switch_state = 0
grovepi.pinMode(button, "INPUT")
button_callback = None
def read_button_state():
if not grove_lock.acquire(timeout=0.05):
return None
try:
return grovepi.digitalRead(button)
except Exception as e:
log(f"BTN Error: {e}")
return None
finally:
grove_lock.release()
def monitor_button():
global button_switch_state
last_button_state = button_switch_state
while True:
time.sleep(0.04)
current_state = read_button_state()
if current_state is not None:
if current_state == 1 and last_button_state == 0:
if button_callback:
button_callback()
last_button_state = current_state
def start_button_monitoring_thread():
threading.Thread(target=monitor_button, daemon=True).start()
def set_callback(callback):
global button_callback
button_callback = callback
+33
View File
@@ -0,0 +1,33 @@
import grovepi
import math
from sensors.lock import grove_lock
from picamera2 import Picamera2, Preview
import time
picam2 = Picamera2()
camera_config = picam2.create_still_configuration()
picam2.configure(camera_config)
picam2.start()
time.sleep(2)
def preview_camera():
picam2.start_preview(Preview.DRM)
def stop_preview_camera():
picam2.stop_preview()
def take_picture():
"""Takes a picture and saves it to the file system"""
picam2.capture_file("test.jpg")
return "test.jpg"
def get_picture():
"""Returns the image bytes as base64
"""
file_path = take_picture()
with open(file_path, "rb") as f:
image_bytes = f.read()
return image_bytes
+122
View File
@@ -0,0 +1,122 @@
import serial
import time
import threading
from shared.logging import log
from sensors.lock import serial_lock
def calculate_nmea_checksum(line: str) -> bool:
"""Validates standard NMEA 0183 sentence checksum ($...*HH)."""
if not line.startswith('$') or '*' not in line:
return False
try:
content, checksum_str = line[1:].split('*', 1)
calculated_checksum = 0
for char in content:
calculated_checksum ^= ord(char)
return calculated_checksum == int(checksum_str[:2], 16)
except Exception:
return False
class GROVEGPS:
def __init__(self, port='/dev/ttyAMA0', baud=9600, timeout=1):
self.ser = serial.Serial(port, baud, timeout=timeout)
self.clean_data()
def clean_data(self):
self.timestamp = ""
self.quality = 0
self.satellites = 0
self.altitude = -1.0
self.latitude = -1.0
self.longitude = -1.0
def read(self):
"""Reads the latest GGA sentence from serial, thread-safely."""
with serial_lock:
# 1. Flush accumulated stale data in the UART buffer
if self.ser.in_waiting > 0:
self.ser.reset_input_buffer()
# 2. Try reading up to 15 lines to catch the freshest GGA sentence
for _ in range(5):
raw_bytes = self.ser.readline()
try:
line = raw_bytes.decode('utf-8', errors='ignore').strip()
# log(f"GPS: Read line: {line}")
except Exception:
continue
# Supports both $GPGGA and modern $GNGGA sentences
if (line.startswith('$GPGGA') or line.startswith('$GNGGA')) and calculate_nmea_checksum(line):
if self.parse_gga(line):
return True
return False
def parse_gga(self, line):
self.clean_data()
gga = line.split(',')
if len(gga) < 10:
return False
try:
self.timestamp = gga[1]
self.quality = int(gga[6]) if gga[6] != "" else 0
self.satellites = int(gga[7]) if gga[7] != "" else 0
# If quality > 0 and coordinates exist, convert NMEA DDDMM.MMMM to decimal degrees
if self.quality > 0 and gga[2] != "" and gga[4] != "":
lat_raw = float(gga[2])
ns = gga[3]
lon_raw = float(gga[4])
ew = gga[5]
# Latitude calculation
lat_deg = lat_raw // 100
lat_min = lat_raw % 100
self.latitude = lat_deg + (lat_min / 60.0)
if ns == 'S':
self.latitude = -self.latitude
# Longitude calculation
lon_deg = lon_raw // 100
lon_min = lon_raw % 100
self.longitude = lon_deg + (lon_min / 60.0)
if ew == 'W':
self.longitude = -self.longitude
self.altitude = float(gga[9]) if gga[9] != "" else -1.0
return True
else:
# No lock on this line
return True
except (ValueError, IndexError):
return False
# Shared instance
gps = GROVEGPS()
def get_gps_data():
"""Returns GPS dictionary if fix is valid, otherwise returns None."""
has_data = gps.read()
# Strictly check that we have a valid GPS lock (quality > 0 and valid coordinates)
if has_data and gps.quality > 0 and gps.latitude != -1.0:
return {
"timestamp": gps.timestamp,
"latitude": round(gps.latitude, 6),
"longitude": round(gps.longitude, 6),
"altitude": gps.altitude,
"quality": gps.quality,
"satellites": gps.satellites
}
else:
log(f"GPS: No valid fix or data available. Satellites: {gps.satellites}, Quality: {gps.quality}")
# Return None so main.py doesn't process or log empty GPS data
return None
+2
View File
@@ -0,0 +1,2 @@
# import orchestrateur.sensors.lib.grovepi_old as grovepi_old
# import sensors.lib.grove_i2c_temp_hum_mini as grove_i2c_temp_hum_mini
@@ -0,0 +1,87 @@
#!/usr/bin/env python
#
# GrovePi Library for using the Grove - Temperature&Humidity Sensor (http://www.seeedstudio.com/depot/Grove-TemperatureHumidity-Sensor-HighAccuracy-Mini-p-1921.html)
#
# The GrovePi connects the Raspberry Pi and Grove sensors. You can learn more about GrovePi here: http://www.dexterindustries.com/GrovePi
#
# Have a question about this library? Ask on the forums here: http://forum.dexterindustries.com/c/grovepi
#
# Released under the MIT license (http://choosealicense.com/licenses/mit/).
# For more information see https://github.com/DexterInd/GrovePi/blob/master/LICENSE
#################################################################################################################################################
# NOTE:
# The software for this sensor is still in development and might make your GrovePi unuable as long as this sensor is connected with the GrovePi
#################################################################################################################################################
import time,sys
import RPi.GPIO as GPIO
import smbus
debug = 0
# use the bus that matches your raspi version
rev = GPIO.RPI_REVISION
if rev == 2 or rev == 3:
bus = smbus.SMBus(1)
else:
bus = smbus.SMBus(0)
class th02:
ADDRESS = 0x40
TH02_REG_STATUS = 0x00
TH02_REG_DATA_H = 0x01
TH02_REG_DATA_L = 0x02
TH02_REG_CONFIG = 0x03
TH02_REG_ID = 0x11
TH02_STATUS_RDY_MASK = 0x01
TH02_CMD_MEASURE_HUMI = [0x01]
TH02_CMD_MEASURE_TEMP = [0x11]
SUCCESS = 0
def getTemperature(self):
bus.write_i2c_block_data(self.ADDRESS, self.TH02_REG_CONFIG, self.TH02_CMD_MEASURE_TEMP)
while 1:
status=self.getStatus()
if debug:
print("st:",status)
if status:
break
t_raw=bus.read_i2c_block_data(self.ADDRESS, self.TH02_REG_DATA_H,3)
if debug:
print(t_raw)
temperature = (t_raw[1]<<8|t_raw[2])>>2
return (temperature/32.0)-50.0
def getHumidity(self):
bus.write_i2c_block_data(self.ADDRESS, self.TH02_REG_CONFIG, self.TH02_CMD_MEASURE_HUMI)
while 1:
status=self.getStatus()
if debug:
print("st:",status)
if status:
break
t_raw=bus.read_i2c_block_data(self.ADDRESS, self.TH02_REG_DATA_H,3)
if debug:
print(t_raw)
temperature = (t_raw[1]<<8|t_raw[2])>>4
return (temperature/16.0)-24.0
def getStatus(self):
status=bus.read_i2c_block_data(self.ADDRESS, self.TH02_REG_STATUS,1)
if debug:
print(status)
if status[0] & self.TH02_STATUS_RDY_MASK != 1:
return 1
else:
return 0
if __name__ == "__main__":
t= th02()
while True:
print(t.getTemperature(),t.getHumidity())
time.sleep(.5)
+691
View File
@@ -0,0 +1,691 @@
#!/usr/bin/env python
#
# GrovePi Python library
# v1.4
#
# This file provides the basic functions for using the GrovePi
#
# The GrovePi connects the Raspberry Pi and Grove sensors. You can learn more about GrovePi here: http://www.dexterindustries.com/GrovePi
#
# Have a question about this example? Ask on the forums here: http://forum.dexterindustries.com/c/grovepi
#
'''
## License
The MIT License (MIT)
GrovePi for the Raspberry Pi: an open source platform for connecting Grove Sensors to the Raspberry Pi.
Copyright (C) 2017 Dexter Industries
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
'''
# Initial Date: 13 Feb 2014
# Last Updated: 11 Nov 2016
# http://www.dexterindustries.com/
# Author Date Comments
# Karan 13 Feb 2014 Initial Authoring
# 11 Nov 2016 I2C retries added for faster IO
# DHT function updated to look for nan's
__version__ = '1.4.1'
import sys
import time
import math
import struct
import numpy
import di_i2c
def set_bus(bus):
global i2c
i2c = di_i2c.DI_I2C(bus = bus, address = address)
address = 0x04
max_recv_size = 10
set_bus("RPI_1SW")
if sys.version_info<(3,0):
p_version = 2
else:
p_version = 3
# Earliest version of the firmware to work with
works_with_firmware = [
"1.4.0"
]
# interrupt operations
COUNT_CHANGES = 0
COUNT_LOW_DURATION = 1
# interrupt trigger mode
CHANGE = 1
FALLING = 2
RISING = 3
# This allows us to be more specific about which commands contain unused bytes
unused = 0
retries = 10
additional_waiting = 0
# Get firmware version
version_cmd = [8]
# No data is available from the GrovePi
data_not_available_cmd = [23]
# Command Format
# digitalRead() command format header
dRead_cmd = [1]
# digitalWrite() command format header
dWrite_cmd = [2]
# analogRead() command format header
aRead_cmd = [3]
# analogWrite() command format header
aWrite_cmd = [4]
# pinMode() command format header
pMode_cmd = [5]
# Ultrasonic read
uRead_cmd = [7]
# Accelerometer (+/- 1.5g) read
acc_xyz_cmd = [20]
# RTC get time
rtc_getTime_cmd = [30]
# DHT Pro sensor temperature
dht_temp_cmd = [40]
# Grove LED Bar commands
# Initialise
ledBarInit_cmd = [50]
# Set orientation
ledBarOrient_cmd = [51]
# Set level
ledBarLevel_cmd = [52]
# Set single LED
ledBarSetOne_cmd = [53]
# Toggle single LED
ledBarToggleOne_cmd = [54]
# Set all LEDs
ledBarSet_cmd = [55]
# Get current state
ledBarGet_cmd = [56]
# Grove 4 Digit Display commands
# Initialise
fourDigitInit_cmd = [70]
# Set brightness, not visible until next cmd
fourDigitBrightness_cmd = [71]
# Set numeric value without leading zeros
fourDigitValue_cmd = [72]
# Set numeric value with leading zeros
fourDigitValueZeros_cmd = [73]
# Set individual digit
fourDigitIndividualDigit_cmd = [74]
# Set individual leds of a segment
fourDigitIndividualLeds_cmd = [75]
# Set left and right values with colon
fourDigitScore_cmd = [76]
# Analog read for n seconds
fourDigitAnalogRead_cmd = [77]
# Entire display on
fourDigitAllOn_cmd = [78]
# Entire display off
fourDigitAllOff_cmd = [79]
# Grove Chainable RGB LED commands
# Store color for later use
storeColor_cmd = [90]
# Initialise
chainableRgbLedInit_cmd = [91]
# Initialise and test with a simple color
chainableRgbLedTest_cmd = [92]
# Set one or more leds to the stored color by pattern
chainableRgbLedSetPattern_cmd = [93]
# set one or more leds to the stored color by modulo
chainableRgbLedSetModulo_cmd = [94]
# sets leds similar to a bar graph, reversible
chainableRgbLedSetLevel_cmd = [95]
# Read the button from IR sensor
ir_read_cmd = [21]
# Set pin for the IR receiver
ir_recv_pin_cmd = [22]
# Check if there's data coming from the IR receiver
ir_read_isdata = [24]
# Interrupt-based devices
isr_set_cmd = [6]
isr_unset_cmd = [9]
isr_read_cmd = [10]
isr_clear_cmd = [11]
isr_active_cmd = [12]
# Grove Encoders
encoder_read_cmd = [13]
encoder_en_cmd = [14]
encoder_dis_cmd = [15]
# Dust, Encoder & Flow Sensor commands
# dust_sensor_read_cmd=[10]
# dust_sensor_en_cmd=[14]
# dust_sensor_dis_cmd=[15]
# dust_sensor_int_cmd=[9]
# dust_sensor_read_int_cmd=[6]
# flow_read_cmd=[12]
# flow_disable_cmd=[13]
# flow_en_cmd=[18]
# Function declarations of the various functions used for encoding and sending
# data from RPi to Arduino
# Write I2C block to the GrovePi
def write_i2c_block(block, custom_timing = None):
'''
Now catches and raises Keyboard Interrupt that the user is responsible to catch.
'''
counter = 0
reg = block[0]
data = block[1:]
while counter < 3:
try:
i2c.write_reg_list(reg, data)
time.sleep(0.002 + additional_waiting)
return
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
counter += 1
time.sleep(0.003)
continue
# Read I2C block from the GrovePi
def read_i2c_block(no_bytes = max_recv_size):
'''
Now catches and raises Keyboard Interrupt that the user is responsible to catch.
'''
data = data_not_available_cmd
counter = 0
while data[0] in [data_not_available_cmd[0], 255] and counter < 3:
try:
data = i2c.read_list(reg = None, len = no_bytes)
time.sleep(0.002 + additional_waiting)
if counter > 0:
counter = 0
except KeyboardInterrupt:
raise KeyboardInterrupt
except:
counter += 1
time.sleep(0.003)
return data
def read_identified_i2c_block(read_command_id, no_bytes):
data = [-1]
while len(data) <= 1:
data = read_i2c_block(no_bytes + 1)
return data[1:]
# Arduino Digital Read
def digitalRead(pin):
write_i2c_block(dRead_cmd + [pin, unused, unused])
data = read_identified_i2c_block( dRead_cmd, no_bytes = 1)[0]
return data
# Arduino Digital Write
def digitalWrite(pin, value):
write_i2c_block(dWrite_cmd + [pin, value, unused])
read_i2c_block(no_bytes = 1)
return 1
# Read analog value from Pin
def analogRead(pin):
write_i2c_block(aRead_cmd + [pin, unused, unused])
number = read_identified_i2c_block(aRead_cmd, no_bytes = 2)
return number[0] * 256 + number[1]
# Write PWM
def analogWrite(pin, value):
write_i2c_block(aWrite_cmd + [pin, value, unused])
read_i2c_block(no_bytes = 1)
return 1
# Setting Up Pin mode on Arduino
def pinMode(pin, mode):
if mode == "OUTPUT":
write_i2c_block(pMode_cmd + [pin, 1, unused])
elif mode == "INPUT":
write_i2c_block(pMode_cmd + [pin, 0, unused])
read_i2c_block(no_bytes = 1)
return 1
# Read temp in Celsius from Grove Temperature Sensor
def temp(pin, model = '1.0'):
# each of the sensor revisions use different thermistors, each with their own B value constant
if model == '1.2':
bValue = 4250 # sensor v1.2 uses thermistor ??? (assuming NCP18WF104F03RC until SeeedStudio clarifies)
elif model == '1.1':
bValue = 4250 # sensor v1.1 uses thermistor NCP18WF104F03RC
else:
bValue = 3975 # sensor v1.0 uses thermistor TTC3A103*39H
a = analogRead(pin)
resistance = (float)(1023 - a) * 10000 / a
t = (float)(1 / (math.log(resistance / 10000) / bValue + 1 / 298.15) - 273.15)
return t
# Read value from Grove Ultrasonic
def ultrasonicRead(pin):
write_i2c_block(uRead_cmd + [pin, unused, unused])
number = read_identified_i2c_block(uRead_cmd, no_bytes = 2)
return (number[0] * 256 + number[1])
# Read the firmware version
def version():
write_i2c_block(version_cmd + [unused, unused, unused])
number = read_identified_i2c_block(version_cmd, no_bytes = 3)
return "%s.%s.%s" % (number[0], number[1], number[2])
# Read Grove Accelerometer (+/- 1.5g) XYZ value
# Need to investigate why this reports what was read with the previous command
# Doesn't look to be implemented on the GrovePi
def acc_xyz():
write_i2c_block(acc_xyz_cmd + [unused, unused, unused])
number = read_identified_i2c_block(acc_xyz_cmd, no_bytes = 3)
if number[1] > 32:
number[1] = - (number[1] - 224)
if number[2] > 32:
number[2] = - (number[2] - 224)
if number[3] > 32:
number[3] = - (number[3] - 224)
return (number[0], number[1], number[2])
# Read from Grove RTC
# Doesn't look to be implemented on the GrovePi
def rtc_getTime():
write_i2c_block(rtc_getTime_cmd + [unused, unused, unused])
number = read_i2c_block()
return number
# Read and return temperature and humidity from Grove DHT Pro
def dht(pin, module_type):
write_i2c_block(dht_temp_cmd + [pin, module_type, unused])
number = read_identified_i2c_block(dht_temp_cmd, no_bytes = 8)
if p_version==2:
h=''
for element in (number[0:4]):
h+=chr(element)
t_val=struct.unpack('f', h)
t = round(t_val[0], 2)
h = ''
for element in (number[4:8]):
h+=chr(element)
hum_val=struct.unpack('f',h)
hum = round(hum_val[0], 2)
else:
t_val=bytearray(number[0:4])
h_val=bytearray(number[4:8])
t=round(struct.unpack('f',t_val)[0],2)
hum=round(struct.unpack('f',h_val)[0],2)
if t > -100.0 and t <150.0 and hum >= 0.0 and hum<=100.0:
return [t, hum]
else:
return [float('nan'),float('nan')]
# Grove - Infrared Receiver - get the commands received from the Grove IR sensor
def ir_read_signal():
write_i2c_block(ir_read_cmd + [unused, unused, unused])
data_back = read_identified_i2c_block(ir_read_cmd, no_bytes = 7)
return (data_back[0],
data_back[1] + data_back[2] * 256,
data_back[3] + data_back[4] * 256 + data_back[5] * (256 ** 2) + data_back[6] * (256 ** 3))
# Grove - Infrared Receiver - set the pin on which the Grove IR sensor is connected
def ir_recv_pin(pin):
write_i2c_block(ir_recv_pin_cmd + [pin, unused, unused])
read_i2c_block(no_bytes = 1)
# Grove - Infrared Receiver - check if there's any data that hasn't been read so far
def ir_is_data():
write_i2c_block(ir_read_isdata + 3 * [unused])
number = read_identified_i2c_block(ir_read_isdata, no_bytes = 1)
return number[0] != 0
# after a list of numerical values is provided
# the function returns a list with the outlier(or extreme) values removed
# make the std_factor_threshold bigger so that filtering becomes less strict
# and make the std_factor_threshold smaller to get the opposite
def statisticalNoiseReduction(values, std_factor_threshold = 2):
if len(values) == 0:
return []
mean = numpy.mean(values)
standard_deviation = numpy.std(values)
if standard_deviation == 0:
return values
filtered_values = [element for element in values if element > mean - std_factor_threshold * standard_deviation]
filtered_values = [element for element in filtered_values if element < mean + std_factor_threshold * standard_deviation]
return filtered_values
# Grove LED Bar - initialise
# orientation: (0 = red to green, 1 = green to red)
def ledBar_init(pin, orientation):
write_i2c_block(ledBarInit_cmd + [pin, orientation, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove LED Bar - set orientation
# orientation: (0 = red to green, 1 = green to red)
def ledBar_orientation(pin, orientation):
write_i2c_block(ledBarOrient_cmd + [pin, orientation, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove LED Bar - set level
# level: (0-10)
def ledBar_setLevel(pin, level):
write_i2c_block(ledBarLevel_cmd + [pin, level, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove LED Bar - set single led
# led: which led (1-10)
# state: off or on (0-1)
def ledBar_setLed(pin, led, state):
write_i2c_block(ledBarSetOne_cmd + [pin, led, state])
read_i2c_block(no_bytes = 1)
return 1
# Grove LED Bar - toggle single led
# led: which led (1-10)
def ledBar_toggleLed(pin, led):
write_i2c_block(ledBarToggleOne_cmd + [pin, led, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove LED Bar - set all leds
# state: (0-1023) or (0x00-0x3FF) or (0b0000000000-0b1111111111) or (int('0000000000',2)-int('1111111111',2))
def ledBar_setBits(pin, state):
byte1 = state & 255
byte2 = state >> 8
write_i2c_block(ledBarSet_cmd + [pin, byte1, byte2])
read_i2c_block(no_bytes = 1)
return 1
# Grove LED Bar - get current state
# state: (0-1023) a bit for each of the 10 LEDs
def ledBar_getBits(pin):
write_i2c_block(ledBarGet_cmd + [pin, unused, unused])
block = read_identified_i2c_block(ledBarGet_cmd, no_bytes = 2)
return block[0] ^ (block[1] << 8)
# Grove 4 Digit Display - initialise
def fourDigit_init(pin):
write_i2c_block(fourDigitInit_cmd + [pin, unused, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove 4 Digit Display - set numeric value with or without leading zeros
# value: (0-65535) or (0000-FFFF)
def fourDigit_number(pin, value, leading_zero):
# split the value into two bytes so we can render 0000-FFFF on the display
byte1 = value & 255
byte2 = value >> 8
# separate commands to overcome current 4 bytes per command limitation
if (leading_zero):
write_i2c_block(fourDigitValue_cmd + [pin, byte1, byte2])
else:
write_i2c_block(fourDigitValueZeros_cmd + [pin, byte1, byte2])
read_i2c_block(no_bytes = 1)
return 1
# Grove 4 Digit Display - set brightness
# brightness: (0-7)
def fourDigit_brightness(pin, brightness):
# not actually visible until next command is executed
write_i2c_block(fourDigitBrightness_cmd + [pin, brightness, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove 4 Digit Display - set individual segment (0-9,A-F)
# segment: (0-3)
# value: (0-15) or (0-F)
def fourDigit_digit(pin, segment, value):
write_i2c_block(fourDigitIndividualDigit_cmd + [pin, segment, value])
read_i2c_block(no_bytes = 1)
return 1
# Grove 4 Digit Display - set 7 individual leds of a segment
# segment: (0-3)
# leds: (0-255) or (0-0xFF) one bit per led, segment 2 is special, 8th bit is the colon
def fourDigit_segment(pin, segment, leds):
write_i2c_block(fourDigitIndividualLeds_cmd + [pin, segment, leds])
read_i2c_block(no_bytes = 1)
return 1
# Grove 4 Digit Display - set left and right values (0-99), with leading zeros and a colon
# left: (0-255) or (0-FF)
# right: (0-255) or (0-FF)
# colon will be lit
def fourDigit_score(pin, left, right):
write_i2c_block(fourDigitScore_cmd + [pin, left, right])
read_i2c_block(no_bytes = 1)
return 1
# Grove 4 Digit Display - display analogRead value for n seconds, 4 samples per second
# analog: analog pin to read
# duration: analog read for this many seconds
def fourDigit_monitor(pin, analog, duration):
write_i2c_block(fourDigitAnalogRead_cmd + [pin, analog, duration])
read_i2c_block(no_bytes = 1)
time.sleep(duration)
return 1
# Grove 4 Digit Display - turn entire display on (88:88)
def fourDigit_on(pin):
write_i2c_block(fourDigitAllOn_cmd + [pin, unused, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove 4 Digit Display - turn entire display off
def fourDigit_off(pin):
write_i2c_block(fourDigitAllOff_cmd + [pin, unused, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove Chainable RGB LED - store a color for later use
# red: 0-255
# green: 0-255
# blue: 0-255
def storeColor(red, green, blue):
write_i2c_block(storeColor_cmd + [red, green, blue])
read_i2c_block(no_bytes = 1)
return 1
# Grove Chainable RGB LED - initialise
# numLeds: how many leds do you have in the chain
def chainableRgbLed_init(pin, numLeds):
write_i2c_block(chainableRgbLedInit_cmd + [pin, numLeds, unused])
read_i2c_block(no_bytes = 1)
return 1
# Grove Chainable RGB LED - initialise and test with a simple color
# numLeds: how many leds do you have in the chain
# testColor: (0-7) 3 bits in total - a bit for red, green and blue, eg. 0x04 == 0b100 (0bRGB) == rgb(255, 0, 0) == #FF0000 == red
# ie. 0 black, 1 blue, 2 green, 3 cyan, 4 red, 5 magenta, 6 yellow, 7 white
def chainableRgbLed_test(pin, numLeds, testColor):
write_i2c_block(chainableRgbLedTest_cmd + [pin, numLeds, testColor])
read_i2c_block(no_bytes = 1)
return 1
# Grove Chainable RGB LED - set one or more leds to the stored color by pattern
# pattern: (0-3) 0 = this led only, 1 all leds except this led, 2 this led and all leds inwards, 3 this led and all leds outwards
# whichLed: index of led you wish to set counting outwards from the GrovePi, 0 = led closest to the GrovePi
def chainableRgbLed_pattern(pin, pattern, whichLed):
write_i2c_block(chainableRgbLedSetPattern_cmd + [pin, pattern, whichLed])
read_i2c_block(no_bytes = 1)
return 1
# Grove Chainable RGB LED - set one or more leds to the stored color by modulo
# offset: index of led you wish to start at, 0 = led closest to the GrovePi, counting outwards
# divisor: when 1 (default) sets stored color on all leds >= offset, when 2 sets every 2nd led >= offset and so on
def chainableRgbLed_modulo(pin, offset, divisor):
write_i2c_block(chainableRgbLedSetModulo_cmd + [pin, offset, divisor])
read_i2c_block(no_bytes = 1)
return 1
# Grove Chainable RGB LED - sets leds similar to a bar graph, reversible
# level: (0-10) the number of leds you wish to set to the stored color
# reversible (0-1) when 0 counting outwards from GrovePi, 0 = led closest to the GrovePi, otherwise counting inwards
def chainableRgbLed_setLevel(pin, level, reverse):
write_i2c_block(chainableRgbLedSetLevel_cmd + [pin, level, reverse])
read_i2c_block(no_bytes = 1)
return 1
def set_pin_interrupt(pin, ftype, interrupt_mode, period):
'''
Attach an interrupt to a pin.
pin - D2-D8 pins
ftype - 0 for COUNT_CHANGES, 1 for COUNT_LOW_DURATION
interrupt_mode - 1 for CHANGE, 2 for FALLING, 3 for RISING
period - as measured in ms (max 65535 ms)
'''
period_high = period >> 8
period_low = period & 0xff
combined_params = (pin & 0x0f) + ((ftype & 0x03) << 4) + ((interrupt_mode & 0x03) << 6)
write_i2c_block(isr_set_cmd + [combined_params, period_high, period_low])
read_i2c_block(no_bytes = 1)
def unset_pin_interrupt(pin):
'''
Detach an interrupt from a pin.
pin - D2-D8 pins
'''
write_i2c_block(isr_unset_cmd + [pin, unused, unused])
read_i2c_block(no_bytes = 1)
def unset_all_interrupts():
'''
Detach all attached interrupts from all D2-D8 pins.
pin - D2-D8 pins
'''
write_i2c_block(isr_clear_cmd + 3 * [unused])
read_i2c_block(no_bytes = 1)
def is_interrupt_active(pin):
write_i2c_block(isr_active_cmd + [pin, unused, unused])
data = read_identified_i2c_block(isr_active_cmd, no_bytes = 2)
value = data[1] >> pin
return value != 0
def get_active_interrupts():
'''
Get list of attached interrupts for a given pin or all of them.
pin - D2-D8 pins; if it's 255 return the state of all pins
'''
pin = 255
write_i2c_block(isr_active_cmd + [pin, unused, unused])
data = read_identified_i2c_block(isr_active_cmd, no_bytes = 2)
value = data[0] + (data[1] << 8)
active_interrupts = [i for i in range(2 * 8) if ((value >> i) & 0x01)]
return active_interrupts
def read_interrupt_state(pin):
'''
Read number of pulses/changes on given port that occurred within a time period.
pin - D2-D8 pins
'''
write_i2c_block(isr_read_cmd + [pin, unused, unused])
data = read_identified_i2c_block(isr_read_cmd, no_bytes = 4)
value = data[0] + (data[1] << 8) + (data[2] << 16) + (data[3] << 24)
return value
def dust_sensor_en(pin = 2, period = 30000):
set_pin_interrupt(pin, ftype=COUNT_LOW_DURATION, interrupt_mode=CHANGE, period=period)
def dust_sensor_dis(pin = 2):
unset_pin_interrupt(pin)
def dust_sensor_read(pin = 2, period = 30000):
'''
By default, the sample rate is set to 1 at every 30 seconds and this
function was written only for that interval.
If you wish to use a different
interval, then use dust_sensor_read_more function. To set a
different interval, use set_dust_sensor_interval function.
'''
lpo = read_interrupt_state(pin)
percentage = 100.0 * lpo / period
concentration = 1.1 * percentage ** 3 - 3.8 * percentage ** 2 + 520 * percentage + 0.62
return lpo, percentage, concentration
def encoder_en(pin = 2, steps = 32):
write_i2c_block(encoder_en_cmd + [pin, steps, unused])
read_i2c_block(no_bytes = 1)
def encoder_dis(pin = 2):
write_i2c_block(encoder_dis_cmd + [pin, unused, unused])
read_i2c_block(no_bytes = 1)
def encoderRead(pin = 2):
write_i2c_block(encoder_read_cmd + [pin, unused, unused])
data = read_identified_i2c_block(encoder_read_cmd, no_bytes = 4)
value = data[0] + (data[1] << 8) + (data[2] << 16) + (data[3] << 24)
return value
def flowEnable(pin = 2, period = 2000):
set_pin_interrupt(pin, ftype=COUNT_CHANGES, interrupt_mode=RISING, period=period)
def flowDisable(pin = 2):
unset_pin_interrupt(pin)
def flowRead(pin = 2):
val = read_interrupt_state(pin)
return val
def main():
print("library supports this fw versions: " +
" ".join('{}'.format(k[1]) for k in enumerate(works_with_firmware)))
if __name__ == "__main__":
main()
+5
View File
@@ -0,0 +1,5 @@
import threading
# Dedicated lock for I2C bus access (used by GrovePi sensors)
grove_lock = threading.Lock()
# Dedicated lock for UART/Serial port access
serial_lock = threading.Lock()
+52
View File
@@ -0,0 +1,52 @@
# from sensors.lib import grove_i2c_temp_hum_mini
# t= grove_i2c_temp_hum_mini.th02()
# def get_temperature():
# """Get the temperature in Celsius from the TH02 sensor."""
# # try:
# return t.getTemperature()
# # except Exception as e:
# # print(f"Error reading temperature: {e}")
# # return None
# def get_humidity():
# """Get the humidity in percentage from the TH02 sensor."""
# # try:
# return t.getHumidity()
# # except Exception as e:
# # print(f"Error reading humidity: {e}")
# # return None
# import seeed_dht
# sensor = seeed_dht.DHT("11", 4) # DHT11 sensor on GPIO pin 4
# def get_humidity_and_temperature():
# humi, temp = sensor.read()
# return humi, temp
# import sensors.lib.grovepi as grovepi
import grovepi
import math
from sensors.lock import grove_lock
# Connect the Grove Temperature & Humidity Sensor Pro to digital port D3
# This example uses the blue colored sensor.
# SIG,NC,VCC,GND
sensor = 3 # The Sensor goes on digital port 3.
# temp_humidity_sensor_type
# Grove Base Kit comes with the blue sensor.
blue = 0 # The Blue colored sensor.
white = 1 # The White colored sensor.
def get_temperature_and_humidity():
with grove_lock:
[temp,humidity] = grovepi.dht(sensor,blue)
if math.isnan(temp) == False and math.isnan(humidity) == False:
return temp, humidity
else:
print("Error reading from DHT sensor")
return None, None
+32
View File
@@ -0,0 +1,32 @@
import grovepi
from sensors.lock import grove_lock
# Connect the Grove Ultrasonic Ranger to digital port D4
# SIG,NC,VCC,GND
ULTRASONIC_RANGER_PORT = 4
def read_ultrasonic_ranger(ultrasonic_ranger=ULTRASONIC_RANGER_PORT):
if not grove_lock.acquire(timeout=1.0):
print("Ultrasonic: Lock acquisition timed out")
return None
try:
return grovepi.ultrasonicRead(ultrasonic_ranger)
except Exception as e:
print(f"Error: {e}")
return None
finally:
grove_lock.release()
def get_dish_height():
"""Returns the height of the dish in centimeters."""
distance = read_ultrasonic_ranger()
if distance is not None:
# Assuming the ultrasonic sensor is mounted at a fixed height above the dish
# and pointing downwards, we can calculate the height of the dish.
# For example, if the sensor is 30 cm above the dish when it's empty:
SENSOR_HEIGHT = 30 # cm
dish_height = SENSOR_HEIGHT - distance
return max(dish_height, 0) # Ensure height is not negative
else:
return None
-2
View File
@@ -1,2 +0,0 @@
paho-mqtt>=1.6,<3
pyserial>=3.5,<4
+1
View File
@@ -3,6 +3,7 @@
import shared.deviceTypes as deviceTypes
import shared.config as config
import shared.payloads as payloads
def get_lora(*args, **kwargs):
from .lora_device import get_lora_device
+6 -3
View File
@@ -1,12 +1,15 @@
DEBUG=True
# LoRa
HEARTBEAT_INTERVAL = 10
HEARTBEAT_INTERVAL = 30
# MQTT
MQTT_BROKER_HOST = "192.168.50.1"
MQTT_TOPIC_HELLO = b"smartwave/hello"
MQTT_TOPIC_SENSOR = b"smartwave/sensor"
MQTT_TOPIC_COOKING = b"smartwave/cooking"
MQTT_KEEPALIVE = 30
USE_TLS = True
MQTT_QOS = 1
MQTT_QOS = 1
# Long because messages are stored into the broker and will be sent when the orchestrator is back online.
MQTT_HELLO_INTERVAL = 30
+6
View File
@@ -0,0 +1,6 @@
from shared.config import DEBUG
def log(message):
"""Log a message to the console if DEBUG is enabled."""
if DEBUG:
print(f"\n{message}")
+39 -7
View File
@@ -13,13 +13,13 @@ except ImportError:
from umqtt.simple import MQTTClient as _MQTTClient
BACKEND_NAME = "umqtt.simple"
IS_MICROPYTHON = True
except ImportError:
try:
from umqtt.robust import MQTTClient as _MQTTClient
BACKEND_NAME = "umqtt.robust"
IS_MICROPYTHON = True
except ImportError as exc:
raise ImportError("No MQTT client found. Expected paho.mqtt or umqtt.") from exc
# except ImportError:
# try:
# from umqtt.robust import MQTTClient as _MQTTClient
# BACKEND_NAME = "umqtt.robust"
# IS_MICROPYTHON = True
except ImportError as exc:
raise ImportError("No MQTT client found. Expected paho.mqtt or umqtt.") from exc
DEFAULT_PORT = 8884
@@ -179,6 +179,38 @@ class BrokerClient:
topic = topic.decode('utf-8')
return client.subscribe(topic, qos=qos)
def unsubscribe(self, topic):
client = self.open()
if IS_MICROPYTHON:
import struct
# Ensure the topic is bytes for writing to the socket
topic_bytes = topic if isinstance(topic, bytes) else topic.encode('utf-8')
# 1. Build the MQTT unsubscribe packet header
pkt = bytearray(b"\xa2\0\0\0")
client.pid += 1
# Packet length is: 2 bytes (PID) + 2 bytes (topic length indicator) + topic string length
struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic_bytes), client.pid)
# 2. Write the packet to the socket
client.sock.write(pkt)
client._send_str(topic_bytes)
# 3. Wait for the UNSUBACK confirmation frame (0xB0) from the broker
while True:
op = client.wait_msg()
if op == 0xB0:
resp = client.sock.read(3)
assert resp[1] == pkt[2] and resp[2] == pkt[3]
return client
return client
if isinstance(topic, bytes):
topic = topic.decode('utf-8')
return client.unsubscribe(topic)
def _on_micropython_message(self, topic, payload):
self._store_message(topic, payload, None, False)
+23
View File
@@ -0,0 +1,23 @@
try:
import ujson as json
except ImportError:
import json
def as_json(data):
"""Convert a dictionary to a JSON string."""
try:
return json.dumps(data)
except Exception as e:
print("[Payloads] Error converting to JSON:", e)
return "{}" # Return an empty JSON object on error
def mqtt_hello(id_microwave):
return as_json({
"id_microwave": id_microwave
})
def mqtt_hello_ack(id_orchestrator, id_microwave):
return as_json({
"id_microwave": id_microwave,
"id_orchestrator": id_orchestrator
})
@@ -1 +0,0 @@
pip
@@ -1,163 +0,0 @@
Metadata-Version: 2.4
Name: urllib3
Version: 2.7.0
Summary: HTTP library with thread-safe connection pooling, file post, and more.
Project-URL: Changelog, https://github.com/urllib3/urllib3/blob/main/CHANGES.rst
Project-URL: Documentation, https://urllib3.readthedocs.io
Project-URL: Code, https://github.com/urllib3/urllib3
Project-URL: Issue tracker, https://github.com/urllib3/urllib3/issues
Author-email: Andrey Petrov <andrey.petrov@shazow.net>
Maintainer-email: Seth Michael Larson <sethmichaellarson@gmail.com>, Quentin Pradet <quentin@pradet.me>, Illia Volochii <illia.volochii@gmail.com>
License-Expression: MIT
License-File: LICENSE.txt
Keywords: filepost,http,httplib,https,pooling,ssl,threadsafe,urllib
Classifier: Environment :: Web Environment
Classifier: Intended Audience :: Developers
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Classifier: Programming Language :: Python :: Free Threading :: 2 - Beta
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Internet :: WWW/HTTP
Classifier: Topic :: Software Development :: Libraries
Requires-Python: >=3.10
Provides-Extra: brotli
Requires-Dist: brotli>=1.2.0; (platform_python_implementation == 'CPython') and extra == 'brotli'
Requires-Dist: brotlicffi>=1.2.0.0; (platform_python_implementation != 'CPython') and extra == 'brotli'
Provides-Extra: h2
Requires-Dist: h2<5,>=4; extra == 'h2'
Provides-Extra: socks
Requires-Dist: pysocks!=1.5.7,<2.0,>=1.5.6; extra == 'socks'
Provides-Extra: zstd
Requires-Dist: backports-zstd>=1.0.0; (python_version < '3.14') and extra == 'zstd'
Description-Content-Type: text/markdown
<h1 align="center">
![urllib3](https://github.com/urllib3/urllib3/raw/main/docs/_static/banner_github.svg)
</h1>
<p align="center">
<a href="https://pypi.org/project/urllib3"><img alt="PyPI Version" src="https://img.shields.io/pypi/v/urllib3.svg?maxAge=86400" /></a>
<a href="https://pypi.org/project/urllib3"><img alt="Python Versions" src="https://img.shields.io/pypi/pyversions/urllib3.svg?maxAge=86400" /></a>
<a href="https://discord.gg/urllib3"><img alt="Join our Discord" src="https://img.shields.io/discord/756342717725933608?color=%237289da&label=discord" /></a>
<a href="https://github.com/urllib3/urllib3/actions?query=workflow%3ACI"><img alt="Coverage Status" src="https://img.shields.io/badge/coverage-100%25-success" /></a>
<a href="https://github.com/urllib3/urllib3/actions/workflows/ci.yml?query=branch%3Amain"><img alt="Build Status on GitHub" src="https://github.com/urllib3/urllib3/actions/workflows/ci.yml/badge.svg?branch:main&workflow:CI" /></a>
<a href="https://urllib3.readthedocs.io"><img alt="Documentation Status" src="https://readthedocs.org/projects/urllib3/badge/?version=latest" /></a><br>
<a href="https://deps.dev/pypi/urllib3"><img alt="OpenSSF Scorecard" src="https://api.securityscorecards.dev/projects/github.com/urllib3/urllib3/badge" /></a>
<a href="https://slsa.dev"><img alt="SLSA 3" src="https://slsa.dev/images/gh-badge-level3.svg" /></a>
<a href="https://bestpractices.coreinfrastructure.org/projects/6227"><img alt="CII Best Practices" src="https://bestpractices.coreinfrastructure.org/projects/6227/badge" /></a>
</p>
urllib3 is a powerful, *user-friendly* HTTP client for Python.
urllib3 brings many critical features that are missing from the Python
standard libraries:
- Thread safety.
- Connection pooling.
- Client-side SSL/TLS verification.
- File uploads with multipart encoding.
- Helpers for retrying requests and dealing with HTTP redirects.
- Support for gzip, deflate, brotli, and zstd encoding.
- Proxy support for HTTP and SOCKS.
- 100% test coverage.
... and many more features, but most importantly: Our maintainers have a 15+
year track record of maintaining urllib3 with the highest code standards and
attention to security and safety.
[Much of the Python ecosystem already uses urllib3](https://urllib3.readthedocs.io/en/stable/#who-uses)
and you should too.
## Installing
urllib3 can be installed with [pip](https://pip.pypa.io):
```bash
$ python -m pip install urllib3
```
Alternatively, you can grab the latest source code from [GitHub](https://github.com/urllib3/urllib3):
```bash
$ git clone https://github.com/urllib3/urllib3.git
$ cd urllib3
$ pip install .
```
## Getting Started
urllib3 is easy to use:
```python3
>>> import urllib3
>>> resp = urllib3.request("GET", "http://httpbin.org/robots.txt")
>>> resp.status
200
>>> resp.data
b"User-agent: *\nDisallow: /deny\n"
```
urllib3 has usage and reference documentation at [urllib3.readthedocs.io](https://urllib3.readthedocs.io).
## Community
urllib3 has a [community Discord channel](https://discord.gg/urllib3) for asking questions and
collaborating with other contributors. Drop by and say hello 👋
## Contributing
urllib3 happily accepts contributions. Please see our
[contributing documentation](https://urllib3.readthedocs.io/en/latest/contributing.html)
for some tips on getting started.
## Security Disclosures
To report a security vulnerability, please use the
[Tidelift security contact](https://tidelift.com/security).
Tidelift will coordinate the fix and disclosure with maintainers.
## Maintainers
Meet our maintainers since 2008:
- Current Lead: [@illia-v](https://github.com/illia-v) (Illia Volochii)
- [@sethmlarson](https://github.com/sethmlarson) (Seth M. Larson)
- [@pquentin](https://github.com/pquentin) (Quentin Pradet)
- [@theacodes](https://github.com/theacodes) (Thea Flowers)
- [@haikuginger](https://github.com/haikuginger) (Jess Shapiro)
- [@lukasa](https://github.com/lukasa) (Cory Benfield)
- [@sigmavirus24](https://github.com/sigmavirus24) (Ian Stapleton Cordasco)
- [@shazow](https://github.com/shazow) (Andrey Petrov)
👋
## Sponsorship
If your company benefits from this library, please consider [sponsoring its
development](https://urllib3.readthedocs.io/en/latest/sponsors.html).
## For Enterprise
Professional support for urllib3 is available as part of the [Tidelift
Subscription][1]. Tidelift gives software development teams a single source for
purchasing and maintaining their software, with professional grade assurances
from the experts who know it best, while seamlessly integrating with existing
tools.
[1]: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme
@@ -1,79 +0,0 @@
urllib3-2.7.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
urllib3-2.7.0.dist-info/METADATA,sha256=ZhR4VtMLu2vtAcbOMybQ9I2E7V8oasF3YzoUhrgurOU,6852
urllib3-2.7.0.dist-info/RECORD,,
urllib3-2.7.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
urllib3-2.7.0.dist-info/licenses/LICENSE.txt,sha256=Ew46ZNX91dCWp1JpRjSn2d8oRGnehuVzIQAmgEHj1oY,1093
urllib3/__init__.py,sha256=JMo1tg1nIV1AeJ2vENC_Txfl0e5h6Gzl9DGVk1rWRbo,6979
urllib3/__pycache__/__init__.cpython-312.pyc,,
urllib3/__pycache__/_base_connection.cpython-312.pyc,,
urllib3/__pycache__/_collections.cpython-312.pyc,,
urllib3/__pycache__/_request_methods.cpython-312.pyc,,
urllib3/__pycache__/_version.cpython-312.pyc,,
urllib3/__pycache__/connection.cpython-312.pyc,,
urllib3/__pycache__/connectionpool.cpython-312.pyc,,
urllib3/__pycache__/exceptions.cpython-312.pyc,,
urllib3/__pycache__/fields.cpython-312.pyc,,
urllib3/__pycache__/filepost.cpython-312.pyc,,
urllib3/__pycache__/poolmanager.cpython-312.pyc,,
urllib3/__pycache__/response.cpython-312.pyc,,
urllib3/_base_connection.py,sha256=HzcSEHexgDrRUr60jNniB2KQjdw97SedD5luRfHtCXg,5580
urllib3/_collections.py,sha256=aOVm2mKilvuvT1efAGtkA7pi65C1NC3HJfpFxvYBS8A,17522
urllib3/_request_methods.py,sha256=gCeF85SO_UU4WoPwYHIoz_tw-eM_EVOkLFp8OFsC7DA,9931
urllib3/_version.py,sha256=-gMrKhsPiOj0XzUezVFa59d1S6QgQiKgQT5gGgyM8-w,520
urllib3/connection.py,sha256=Zos3qxKDW9-GQ6aVqfBQVRM5soB0IjHxY_xXWpJaFTI,42786
urllib3/connectionpool.py,sha256=sGFnddXYwlx7KC4JCP1gKvdNGLNK-YTCJDdGACGj3Y8,44164
urllib3/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
urllib3/contrib/__pycache__/__init__.cpython-312.pyc,,
urllib3/contrib/__pycache__/pyopenssl.cpython-312.pyc,,
urllib3/contrib/__pycache__/socks.cpython-312.pyc,,
urllib3/contrib/emscripten/__init__.py,sha256=wyXve8rmqX7s2KqRQBxD5Wl48jzWPn5-1u_XoQBELVc,836
urllib3/contrib/emscripten/__pycache__/__init__.cpython-312.pyc,,
urllib3/contrib/emscripten/__pycache__/connection.cpython-312.pyc,,
urllib3/contrib/emscripten/__pycache__/fetch.cpython-312.pyc,,
urllib3/contrib/emscripten/__pycache__/request.cpython-312.pyc,,
urllib3/contrib/emscripten/__pycache__/response.cpython-312.pyc,,
urllib3/contrib/emscripten/connection.py,sha256=giElsBoUsKVURbZzb8GCrJmqW23Xnvj2aNyQVF42slg,8960
urllib3/contrib/emscripten/emscripten_fetch_worker.js,sha256=z1k3zZ4_hDKd3-tN7wzz8LHjHC2pxN_uu8B3k9D9A3c,3677
urllib3/contrib/emscripten/fetch.py,sha256=5xcd--viFxZd2nBy0aK73dtJ9Tsh1yYZU_SUXwnwibk,23520
urllib3/contrib/emscripten/request.py,sha256=mL28szy1KvE3NJhWor5jNmarp8gwplDU-7gwGZY5g0Q,566
urllib3/contrib/emscripten/response.py,sha256=CDpY0GFoluR3IxECkM2gH5uDfYDM0EL1FWr7B76wtgM,9719
urllib3/contrib/pyopenssl.py,sha256=XZY-QzyT7s8d43qisA4o-eSZYBeIkPyBMZhtj2kkLZs,19734
urllib3/contrib/socks.py,sha256=rSuklfha4-vto-8qSLhSPmf5lmRPIYuqdAxKe9bg2RA,7639
urllib3/exceptions.py,sha256=hQPHoqo4yw46esNSVkciVQXVUgAxWAglsh6MbyQta-Q,9945
urllib3/fields.py,sha256=aGLFAVZpVU-FbJlllve4Ahg0-pH9ZZBQ2Iz7lfpkjkM,10801
urllib3/filepost.py,sha256=U8eNZ-mpKKHhrlbHEEiTxxgK16IejhEa7uz42yqA_dI,2388
urllib3/http2/__init__.py,sha256=xzrASH7R5ANRkPJOot5lGnATOq3KKuyXzI42rcnwmqs,1741
urllib3/http2/__pycache__/__init__.cpython-312.pyc,,
urllib3/http2/__pycache__/connection.cpython-312.pyc,,
urllib3/http2/__pycache__/probe.cpython-312.pyc,,
urllib3/http2/connection.py,sha256=bHMH6fNvatwXPrKqrcn74yA3pUWcqPDppnK1LcKCbP8,12578
urllib3/http2/probe.py,sha256=nnAkqbhAakOiF75rz7W0udZ38Eeh_uD8fjV74N73FEI,3014
urllib3/poolmanager.py,sha256=c0rh0rcUC1t5tDGuUeGIgAzlErasPOwWwLiB67lX8pM,23895
urllib3/py.typed,sha256=UaCuPFa3H8UAakbt-5G8SPacldTOGvJv18pPjUJ5gDY,93
urllib3/response.py,sha256=9SX4BkkdoLgsx1ne6hpt-5PncrnDzPzeqC1DaShSc-M,53219
urllib3/util/__init__.py,sha256=-qeS0QceivazvBEKDNFCAI-6ACcdDOE4TMvo7SLNlAQ,1001
urllib3/util/__pycache__/__init__.cpython-312.pyc,,
urllib3/util/__pycache__/connection.cpython-312.pyc,,
urllib3/util/__pycache__/proxy.cpython-312.pyc,,
urllib3/util/__pycache__/request.cpython-312.pyc,,
urllib3/util/__pycache__/response.cpython-312.pyc,,
urllib3/util/__pycache__/retry.cpython-312.pyc,,
urllib3/util/__pycache__/ssl_.cpython-312.pyc,,
urllib3/util/__pycache__/ssl_match_hostname.cpython-312.pyc,,
urllib3/util/__pycache__/ssltransport.cpython-312.pyc,,
urllib3/util/__pycache__/timeout.cpython-312.pyc,,
urllib3/util/__pycache__/url.cpython-312.pyc,,
urllib3/util/__pycache__/util.cpython-312.pyc,,
urllib3/util/__pycache__/wait.cpython-312.pyc,,
urllib3/util/connection.py,sha256=JjO722lzHlzLXPTkr9ZWBdhseXnMVjMSb1DJLVrXSnQ,4444
urllib3/util/proxy.py,sha256=seP8-Q5B6bB0dMtwPj-YcZZQ30vHuLqRu-tI0JZ2fzs,1148
urllib3/util/request.py,sha256=itpnC8ug7D4nVfDmGUCRMlgkARUQ13r_XMxSnzTwmpE,8363
urllib3/util/response.py,sha256=vQE639uoEhj1vpjEdxu5lNIhJCSUZkd7pqllUI0BZOA,3374
urllib3/util/retry.py,sha256=2YnSX-_FecMShD61Mx5s68J0_btUHZrrc_BkFVRS1P4,19577
urllib3/util/ssl_.py,sha256=Oqe3rIhUU3e3GVgZob2hxmR8Q0ZDhrhESPlPP_GLaVQ,17742
urllib3/util/ssl_match_hostname.py,sha256=Ft44KJzTzGMmKff_ZXP91li2V4WhmvEy16PKBcv4vZk,5479
urllib3/util/ssltransport.py,sha256=Ez4O8pR_vT8dan_FvqBYS6dgDfBXEMfVfrzcdUoWfi4,8847
urllib3/util/timeout.py,sha256=4eT1FVeZZU7h7mYD1Jq2OXNe4fxekdNvhoWUkZusRpA,10346
urllib3/util/url.py,sha256=WRh-TMYXosmgp8m8lT4H5spoHw5yUjlcMCfU53AkoAs,15205
urllib3/util/util.py,sha256=j3lbZK1jPyiwD34T8IgJzdWEZVT-4E-0vYIJi9UjeNA,1146
urllib3/util/wait.py,sha256=_ph8IrUR3sqPqi0OopQgJUlH4wzkGeM5CiyA7XGGtmI,4423
@@ -1,4 +0,0 @@
Wheel-Version: 1.0
Generator: hatchling 1.29.0
Root-Is-Purelib: true
Tag: py3-none-any
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2008-2020 Andrey Petrov and contributors.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -1,49 +1,40 @@
"""
Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more
"""
from __future__ import annotations
from __future__ import absolute_import
# Set default logging handler to avoid "No handler found" warnings.
import logging
import sys
import typing
import warnings
from logging import NullHandler
from . import exceptions
from ._base_connection import _TYPE_BODY
from ._collections import HTTPHeaderDict
from ._version import __version__
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
from .filepost import encode_multipart_formdata
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
from .response import BaseHTTPResponse, HTTPResponse
from .response import HTTPResponse
from .util.request import make_headers
from .util.retry import Retry
from .util.timeout import Timeout
from .util.url import get_host
# Ensure that Python is compiled with OpenSSL 1.1.1+
# If the 'ssl' module isn't available at all that's
# fine, we only care if the module is available.
# === NOTE TO REPACKAGERS AND VENDORS ===
# Please delete this block, this logic is only
# for urllib3 being distributed via PyPI.
# See: https://github.com/urllib3/urllib3/issues/2680
try:
import ssl
import urllib3_secure_extra # type: ignore # noqa: F401
except ImportError:
pass
else:
if not ssl.OPENSSL_VERSION.startswith("OpenSSL "): # Defensive:
warnings.warn(
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
"See: https://github.com/urllib3/urllib3/issues/3020",
exceptions.NotOpenSSLWarning,
)
elif ssl.OPENSSL_VERSION_INFO < (1, 1, 1): # Defensive:
raise ImportError(
"urllib3 v2 only supports OpenSSL 1.1.1+, currently "
f"the 'ssl' module is compiled with {ssl.OPENSSL_VERSION!r}. "
"See: https://github.com/urllib3/urllib3/issues/2168"
)
warnings.warn(
"'urllib3[secure]' extra is deprecated and will be removed "
"in a future release of urllib3 2.x. Read more in this issue: "
"https://github.com/urllib3/urllib3/issues/2680",
category=DeprecationWarning,
stacklevel=2,
)
__author__ = "Andrey Petrov (andrey.petrov@shazow.net)"
__license__ = "MIT"
@@ -51,7 +42,6 @@ __version__ = __version__
__all__ = (
"HTTPConnectionPool",
"HTTPHeaderDict",
"HTTPSConnectionPool",
"PoolManager",
"ProxyManager",
@@ -62,18 +52,15 @@ __all__ = (
"connection_from_url",
"disable_warnings",
"encode_multipart_formdata",
"get_host",
"make_headers",
"proxy_from_url",
"request",
"BaseHTTPResponse",
)
logging.getLogger(__name__).addHandler(NullHandler())
def add_stderr_logger(
level: int = logging.DEBUG,
) -> logging.StreamHandler[typing.TextIO]:
def add_stderr_logger(level=logging.DEBUG):
"""
Helper for quickly adding a StreamHandler to the logger. Useful for
debugging.
@@ -100,112 +87,16 @@ del NullHandler
# mechanisms to silence them.
# SecurityWarning's always go off by default.
warnings.simplefilter("always", exceptions.SecurityWarning, append=True)
# SubjectAltNameWarning's should go off once per host
warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True)
# InsecurePlatformWarning's don't vary between requests, so we keep it default.
warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True)
# SNIMissingWarnings should go off only once.
warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True)
def disable_warnings(category: type[Warning] = exceptions.HTTPWarning) -> None:
def disable_warnings(category=exceptions.HTTPWarning):
"""
Helper for quickly disabling all urllib3 warnings.
"""
warnings.simplefilter("ignore", category)
_DEFAULT_POOL = PoolManager()
def request(
method: str,
url: str,
*,
body: _TYPE_BODY | None = None,
fields: _TYPE_FIELDS | None = None,
headers: typing.Mapping[str, str] | None = None,
preload_content: bool | None = True,
decode_content: bool | None = True,
redirect: bool | None = True,
retries: Retry | bool | int | None = None,
timeout: Timeout | float | int | None = 3,
json: typing.Any | None = None,
) -> BaseHTTPResponse:
"""
A convenience, top-level request method. It uses a module-global ``PoolManager`` instance.
Therefore, its side effects could be shared across dependencies relying on it.
To avoid side effects create a new ``PoolManager`` instance and use it instead.
The method does not accept low-level ``**urlopen_kw`` keyword arguments.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param fields:
Data to encode and send in the request body.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param json:
Data to encode and send as JSON with UTF-encoded in the request body.
The ``"Content-Type"`` header will be set to ``"application/json"``
unless specified otherwise.
"""
return _DEFAULT_POOL.request(
method,
url,
body=body,
fields=fields,
headers=headers,
preload_content=preload_content,
decode_content=decode_content,
redirect=redirect,
retries=retries,
timeout=timeout,
json=json,
)
if sys.platform == "emscripten":
from .contrib.emscripten import inject_into_urllib3 # noqa: 401
inject_into_urllib3()
@@ -1,167 +0,0 @@
from __future__ import annotations
import typing
from .util.connection import _TYPE_SOCKET_OPTIONS
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
from .util.url import Url
_TYPE_BODY = typing.Union[
bytes, typing.IO[typing.Any], typing.Iterable[bytes | str], str
]
class ProxyConfig(typing.NamedTuple):
ssl_context: ssl.SSLContext | None
use_forwarding_for_https: bool
assert_hostname: None | str | typing.Literal[False]
assert_fingerprint: str | None
class _ResponseOptions(typing.NamedTuple):
# TODO: Remove this in favor of a better
# HTTP request/response lifecycle tracking.
request_method: str
request_url: str
preload_content: bool
decode_content: bool
enforce_content_length: bool
if typing.TYPE_CHECKING:
import ssl
from typing import Protocol
from .response import BaseHTTPResponse
class BaseHTTPConnection(Protocol):
default_port: typing.ClassVar[int]
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
host: str
port: int
timeout: None | (
float
) # Instance doesn't store _DEFAULT_TIMEOUT, must be resolved.
blocksize: int
source_address: tuple[str, int] | None
socket_options: _TYPE_SOCKET_OPTIONS | None
proxy: Url | None
proxy_config: ProxyConfig | None
is_verified: bool
proxy_is_verified: bool | None
def __init__(
self,
host: str,
port: int | None = None,
*,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
blocksize: int = 8192,
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
proxy: Url | None = None,
proxy_config: ProxyConfig | None = None,
) -> None: ...
def set_tunnel(
self,
host: str,
port: int | None = None,
headers: typing.Mapping[str, str] | None = None,
scheme: str = "http",
) -> None: ...
def connect(self) -> None: ...
def request(
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
# We know *at least* botocore is depending on the order of the
# first 3 parameters so to be safe we only mark the later ones
# as keyword-only to ensure we have space to extend.
*,
chunked: bool = False,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> None: ...
def getresponse(self) -> BaseHTTPResponse: ...
def close(self) -> None: ...
@property
def is_closed(self) -> bool:
"""Whether the connection either is brand new or has been previously closed.
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
properties must be False.
"""
@property
def is_connected(self) -> bool:
"""Whether the connection is actively connected to any origin (proxy or target)"""
@property
def has_connected_to_proxy(self) -> bool:
"""Whether the connection has successfully connected to its proxy.
This returns False if no proxy is in use. Used to determine whether
errors are coming from the proxy layer or from tunnelling to the target origin.
"""
class BaseHTTPSConnection(BaseHTTPConnection, Protocol):
default_port: typing.ClassVar[int]
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
# Certificate verification methods
cert_reqs: int | str | None
assert_hostname: None | str | typing.Literal[False]
assert_fingerprint: str | None
ssl_context: ssl.SSLContext | None
# Trusted CAs
ca_certs: str | None
ca_cert_dir: str | None
ca_cert_data: None | str | bytes
# TLS version
ssl_minimum_version: int | None
ssl_maximum_version: int | None
ssl_version: int | str | None # Deprecated
# Client certificates
cert_file: str | None
key_file: str | None
key_password: str | None
def __init__(
self,
host: str,
port: int | None = None,
*,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
blocksize: int = 16384,
socket_options: _TYPE_SOCKET_OPTIONS | None = ...,
proxy: Url | None = None,
proxy_config: ProxyConfig | None = None,
cert_reqs: int | str | None = None,
assert_hostname: None | str | typing.Literal[False] = None,
assert_fingerprint: str | None = None,
server_hostname: str | None = None,
ssl_context: ssl.SSLContext | None = None,
ca_certs: str | None = None,
ca_cert_dir: str | None = None,
ca_cert_data: None | str | bytes = None,
ssl_minimum_version: int | None = None,
ssl_maximum_version: int | None = None,
ssl_version: int | str | None = None, # Deprecated
cert_file: str | None = None,
key_file: str | None = None,
key_password: str | None = None,
) -> None: ...
@@ -1,66 +1,34 @@
from __future__ import annotations
from __future__ import absolute_import
try:
from collections.abc import Mapping, MutableMapping
except ImportError:
from collections import Mapping, MutableMapping
try:
from threading import RLock
except ImportError: # Platform-specific: No threads available
class RLock:
def __enter__(self):
pass
def __exit__(self, exc_type, exc_value, traceback):
pass
import typing
from collections import OrderedDict
from enum import Enum, auto
from threading import RLock
if typing.TYPE_CHECKING:
# We can only import Protocol if TYPE_CHECKING because it's a development
# dependency, and is not available at runtime.
from typing import Protocol
from typing_extensions import Self
class HasGettableStringKeys(Protocol):
def keys(self) -> typing.Iterator[str]: ...
def __getitem__(self, key: str) -> str: ...
from .exceptions import InvalidHeader
from .packages import six
from .packages.six import iterkeys, itervalues
__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"]
# Key type
_KT = typing.TypeVar("_KT")
# Value type
_VT = typing.TypeVar("_VT")
# Default type
_DT = typing.TypeVar("_DT")
ValidHTTPHeaderSource = typing.Union[
"HTTPHeaderDict",
typing.Mapping[str, str],
typing.Iterable[tuple[str, str]],
"HasGettableStringKeys",
]
_Null = object()
class _Sentinel(Enum):
not_passed = auto()
def ensure_can_construct_http_header_dict(
potential: object,
) -> ValidHTTPHeaderSource | None:
if isinstance(potential, HTTPHeaderDict):
return potential
elif isinstance(potential, typing.Mapping):
# Full runtime checking of the contents of a Mapping is expensive, so for the
# purposes of typechecking, we assume that any Mapping is the right shape.
return typing.cast(typing.Mapping[str, str], potential)
elif isinstance(potential, typing.Iterable):
# Similarly to Mapping, full runtime checking of the contents of an Iterable is
# expensive, so for the purposes of typechecking, we assume that any Iterable
# is the right shape.
return typing.cast(typing.Iterable[tuple[str, str]], potential)
elif hasattr(potential, "keys") and hasattr(potential, "__getitem__"):
return typing.cast("HasGettableStringKeys", potential)
else:
return None
class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT, _VT]):
class RecentlyUsedContainer(MutableMapping):
"""
Provides a thread-safe dict-like container which maintains up to
``maxsize`` keys while throwing away the least-recently-used keys beyond
@@ -74,134 +42,69 @@ class RecentlyUsedContainer(typing.Generic[_KT, _VT], typing.MutableMapping[_KT,
``dispose_func(value)`` is called. Callback which will get called
"""
_container: typing.OrderedDict[_KT, _VT]
_maxsize: int
dispose_func: typing.Callable[[_VT], None] | None
lock: RLock
ContainerCls = OrderedDict
def __init__(
self,
maxsize: int = 10,
dispose_func: typing.Callable[[_VT], None] | None = None,
) -> None:
super().__init__()
def __init__(self, maxsize=10, dispose_func=None):
self._maxsize = maxsize
self.dispose_func = dispose_func
self._container = OrderedDict()
self._container = self.ContainerCls()
self.lock = RLock()
def __getitem__(self, key: _KT) -> _VT:
def __getitem__(self, key):
# Re-insert the item, moving it to the end of the eviction line.
with self.lock:
item = self._container.pop(key)
self._container[key] = item
return item
def __setitem__(self, key: _KT, value: _VT) -> None:
evicted_item = None
def __setitem__(self, key, value):
evicted_value = _Null
with self.lock:
# Possibly evict the existing value of 'key'
try:
# If the key exists, we'll overwrite it, which won't change the
# size of the pool. Because accessing a key should move it to
# the end of the eviction line, we pop it out first.
evicted_item = key, self._container.pop(key)
self._container[key] = value
except KeyError:
# When the key does not exist, we insert the value first so that
# evicting works in all cases, including when self._maxsize is 0
self._container[key] = value
if len(self._container) > self._maxsize:
# If we didn't evict an existing value, and we've hit our maximum
# size, then we have to evict the least recently used item from
# the beginning of the container.
evicted_item = self._container.popitem(last=False)
evicted_value = self._container.get(key, _Null)
self._container[key] = value
# After releasing the lock on the pool, dispose of any evicted value.
if evicted_item is not None and self.dispose_func:
_, evicted_value = evicted_item
# If we didn't evict an existing value, we might have to evict the
# least recently used item from the beginning of the container.
if len(self._container) > self._maxsize:
_key, evicted_value = self._container.popitem(last=False)
if self.dispose_func and evicted_value is not _Null:
self.dispose_func(evicted_value)
def __delitem__(self, key: _KT) -> None:
def __delitem__(self, key):
with self.lock:
value = self._container.pop(key)
if self.dispose_func:
self.dispose_func(value)
def __len__(self) -> int:
def __len__(self):
with self.lock:
return len(self._container)
def __iter__(self) -> typing.NoReturn:
def __iter__(self):
raise NotImplementedError(
"Iteration over this class is unlikely to be threadsafe."
)
def clear(self) -> None:
def clear(self):
with self.lock:
# Copy pointers to all values, then wipe the mapping
values = list(self._container.values())
values = list(itervalues(self._container))
self._container.clear()
if self.dispose_func:
for value in values:
self.dispose_func(value)
def keys(self) -> set[_KT]: # type: ignore[override]
def keys(self):
with self.lock:
return set(self._container.keys())
return list(iterkeys(self._container))
class HTTPHeaderDictItemView(set[tuple[str, str]]):
"""
HTTPHeaderDict is unusual for a Mapping[str, str] in that it has two modes of
address.
If we directly try to get an item with a particular name, we will get a string
back that is the concatenated version of all the values:
>>> d['X-Header-Name']
'Value1, Value2, Value3'
However, if we iterate over an HTTPHeaderDict's items, we will optionally combine
these values based on whether combine=True was called when building up the dictionary
>>> d = HTTPHeaderDict({"A": "1", "B": "foo"})
>>> d.add("A", "2", combine=True)
>>> d.add("B", "bar")
>>> list(d.items())
[
('A', '1, 2'),
('B', 'foo'),
('B', 'bar'),
]
This class conforms to the interface required by the MutableMapping ABC while
also giving us the nonstandard iteration behavior we want; items with duplicate
keys, ordered by time of first insertion.
"""
_headers: HTTPHeaderDict
def __init__(self, headers: HTTPHeaderDict) -> None:
self._headers = headers
def __len__(self) -> int:
return len(list(self._headers.iteritems()))
def __iter__(self) -> typing.Iterator[tuple[str, str]]:
return self._headers.iteritems()
def __contains__(self, item: object) -> bool:
if isinstance(item, tuple) and len(item) == 2:
passed_key, passed_val = item
if isinstance(passed_key, str) and isinstance(passed_val, str):
return self._headers._has_value_for_header(passed_key, passed_val)
return False
class HTTPHeaderDict(typing.MutableMapping[str, str]):
class HTTPHeaderDict(MutableMapping):
"""
:param headers:
An iterable of field-value pairs. Must not contain multiple field names
@@ -235,11 +138,9 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
'7'
"""
_container: typing.MutableMapping[str, list[str]]
def __init__(self, headers: ValidHTTPHeaderSource | None = None, **kwargs: str):
super().__init__()
self._container = {} # 'dict' is insert-ordered
def __init__(self, headers=None, **kwargs):
super(HTTPHeaderDict, self).__init__()
self._container = OrderedDict()
if headers is not None:
if isinstance(headers, HTTPHeaderDict):
self._copy_from(headers)
@@ -248,155 +149,126 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
if kwargs:
self.extend(kwargs)
def __setitem__(self, key: str, val: str) -> None:
# avoid a bytes/str comparison by decoding before httplib
if isinstance(key, bytes):
key = key.decode("latin-1")
def __setitem__(self, key, val):
self._container[key.lower()] = [key, val]
return self._container[key.lower()]
def __getitem__(self, key: str) -> str:
if isinstance(key, bytes):
key = key.decode("latin-1")
def __getitem__(self, key):
val = self._container[key.lower()]
return ", ".join(val[1:])
def __delitem__(self, key: str) -> None:
if isinstance(key, bytes):
key = key.decode("latin-1")
def __delitem__(self, key):
del self._container[key.lower()]
def __contains__(self, key: object) -> bool:
if isinstance(key, bytes):
key = key.decode("latin-1")
if isinstance(key, str):
return key.lower() in self._container
return False
def __contains__(self, key):
return key.lower() in self._container
def setdefault(self, key: str, default: str = "") -> str:
return super().setdefault(key, default)
def __eq__(self, other: object) -> bool:
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
def __eq__(self, other):
if not isinstance(other, Mapping) and not hasattr(other, "keys"):
return False
else:
other_as_http_header_dict = type(self)(maybe_constructable)
if not isinstance(other, type(self)):
other = type(self)(other)
return dict((k.lower(), v) for k, v in self.itermerged()) == dict(
(k.lower(), v) for k, v in other.itermerged()
)
return {k.lower(): v for k, v in self.itermerged()} == {
k.lower(): v for k, v in other_as_http_header_dict.itermerged()
}
def __ne__(self, other: object) -> bool:
def __ne__(self, other):
return not self.__eq__(other)
def __len__(self) -> int:
if six.PY2: # Python 2
iterkeys = MutableMapping.iterkeys
itervalues = MutableMapping.itervalues
__marker = object()
def __len__(self):
return len(self._container)
def __iter__(self) -> typing.Iterator[str]:
def __iter__(self):
# Only provide the originally cased names
for vals in self._container.values():
yield vals[0]
def discard(self, key: str) -> None:
def pop(self, key, default=__marker):
"""D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
If key is not found, d is returned if given, otherwise KeyError is raised.
"""
# Using the MutableMapping function directly fails due to the private marker.
# Using ordinary dict.pop would expose the internal structures.
# So let's reinvent the wheel.
try:
value = self[key]
except KeyError:
if default is self.__marker:
raise
return default
else:
del self[key]
return value
def discard(self, key):
try:
del self[key]
except KeyError:
pass
def add(self, key: str, val: str, *, combine: bool = False) -> None:
def add(self, key, val):
"""Adds a (name, value) pair, doesn't overwrite the value if it already
exists.
If this is called with combine=True, instead of adding a new header value
as a distinct item during iteration, this will instead append the value to
any existing header value with a comma. If no existing header value exists
for the key, then the value will simply be added, ignoring the combine parameter.
>>> headers = HTTPHeaderDict(foo='bar')
>>> headers.add('Foo', 'baz')
>>> headers['foo']
'bar, baz'
>>> list(headers.items())
[('foo', 'bar'), ('foo', 'baz')]
>>> headers.add('foo', 'quz', combine=True)
>>> list(headers.items())
[('foo', 'bar, baz, quz')]
"""
# avoid a bytes/str comparison by decoding before httplib
if isinstance(key, bytes):
key = key.decode("latin-1")
key_lower = key.lower()
new_vals = [key, val]
# Keep the common case aka no item present as fast as possible
vals = self._container.setdefault(key_lower, new_vals)
if new_vals is not vals:
# if there are values here, then there is at least the initial
# key/value pair
assert len(vals) >= 2
if combine:
vals[-1] = vals[-1] + ", " + val
else:
vals.append(val)
vals.append(val)
def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None:
def extend(self, *args, **kwargs):
"""Generic import function for any type of header-like object.
Adapted version of MutableMapping.update in order to insert items
with self.add instead of self.__setitem__
"""
if len(args) > 1:
raise TypeError(
f"extend() takes at most 1 positional arguments ({len(args)} given)"
"extend() takes at most 1 positional "
"arguments ({0} given)".format(len(args))
)
other = args[0] if len(args) >= 1 else ()
if isinstance(other, HTTPHeaderDict):
for key, val in other.iteritems():
self.add(key, val)
elif isinstance(other, typing.Mapping):
for key, val in other.items():
self.add(key, val)
elif isinstance(other, typing.Iterable):
for key, value in other:
self.add(key, value)
elif hasattr(other, "keys") and hasattr(other, "__getitem__"):
# THIS IS NOT A TYPESAFE BRANCH
# In this branch, the object has a `keys` attr but is not a Mapping or any of
# the other types indicated in the method signature. We do some stuff with
# it as though it partially implements the Mapping interface, but we're not
# doing that stuff safely AT ALL.
elif isinstance(other, Mapping):
for key in other:
self.add(key, other[key])
elif hasattr(other, "keys"):
for key in other.keys():
self.add(key, other[key])
else:
for key, value in other:
self.add(key, value)
for key, value in kwargs.items():
self.add(key, value)
@typing.overload
def getlist(self, key: str) -> list[str]: ...
@typing.overload
def getlist(self, key: str, default: _DT) -> list[str] | _DT: ...
def getlist(
self, key: str, default: _Sentinel | _DT = _Sentinel.not_passed
) -> list[str] | _DT:
def getlist(self, key, default=__marker):
"""Returns a list of all the values for the named field. Returns an
empty list if the key doesn't exist."""
if isinstance(key, bytes):
key = key.decode("latin-1")
try:
vals = self._container[key.lower()]
except KeyError:
if default is _Sentinel.not_passed:
# _DT is unbound; empty list is instance of List[str]
if default is self.__marker:
return []
# _DT is bound; default is instance of _DT
return default
else:
# _DT may or may not be bound; vals[1:] is instance of List[str], which
# meets our external interface requirement of `Union[List[str], _DT]`.
return vals[1:]
def _prepare_for_method_change(self) -> Self:
def _prepare_for_method_change(self):
"""
Remove content-specific header fields before changing the request
method to GET or HEAD according to RFC 9110, Section 15.4.
@@ -422,65 +294,62 @@ class HTTPHeaderDict(typing.MutableMapping[str, str]):
# Backwards compatibility for http.cookiejar
get_all = getlist
def __repr__(self) -> str:
return f"{type(self).__name__}({dict(self.itermerged())})"
def __repr__(self):
return "%s(%s)" % (type(self).__name__, dict(self.itermerged()))
def _copy_from(self, other: HTTPHeaderDict) -> None:
def _copy_from(self, other):
for key in other:
val = other.getlist(key)
self._container[key.lower()] = [key, *val]
if isinstance(val, list):
# Don't need to convert tuples
val = list(val)
self._container[key.lower()] = [key] + val
def copy(self) -> Self:
def copy(self):
clone = type(self)()
clone._copy_from(self)
return clone
def iteritems(self) -> typing.Iterator[tuple[str, str]]:
def iteritems(self):
"""Iterate over all header lines, including duplicate ones."""
for key in self:
vals = self._container[key.lower()]
for val in vals[1:]:
yield vals[0], val
def itermerged(self) -> typing.Iterator[tuple[str, str]]:
def itermerged(self):
"""Iterate over all headers, merging duplicate ones together."""
for key in self:
val = self._container[key.lower()]
yield val[0], ", ".join(val[1:])
def items(self) -> HTTPHeaderDictItemView: # type: ignore[override]
return HTTPHeaderDictItemView(self)
def items(self):
return list(self.iteritems())
def _has_value_for_header(self, header_name: str, potential_value: str) -> bool:
if header_name in self:
return potential_value in self._container[header_name.lower()][1:]
return False
@classmethod
def from_httplib(cls, message): # Python 2
"""Read headers from a Python 2 httplib message object."""
# python2.7 does not expose a proper API for exporting multiheaders
# efficiently. This function re-reads raw lines from the message
# object and extracts the multiheaders properly.
obs_fold_continued_leaders = (" ", "\t")
headers = []
def __ior__(self, other: object) -> HTTPHeaderDict:
# Supports extending a header dict in-place using operator |=
# combining items with add instead of __setitem__
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
return NotImplemented
self.extend(maybe_constructable)
return self
for line in message.headers:
if line.startswith(obs_fold_continued_leaders):
if not headers:
# We received a header line that starts with OWS as described
# in RFC-7230 S3.2.4. This indicates a multiline header, but
# there exists no previous header to which we can attach it.
raise InvalidHeader(
"Header continuation with no previous header: %s" % line
)
else:
key, value = headers[-1]
headers[-1] = (key, value + " " + line.strip())
continue
def __or__(self, other: object) -> Self:
# Supports merging header dicts using operator |
# combining items with add instead of __setitem__
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
return NotImplemented
result = self.copy()
result.extend(maybe_constructable)
return result
key, value = line.split(":", 1)
headers.append((key, value.strip()))
def __ror__(self, other: object) -> Self:
# Supports merging header dicts using operator | when other is on left side
# combining items with add instead of __setitem__
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
return NotImplemented
result = type(self)(maybe_constructable)
result.extend(self)
return result
return cls(headers)
@@ -1,278 +0,0 @@
from __future__ import annotations
import json as _json
import typing
from urllib.parse import urlencode
from ._base_connection import _TYPE_BODY
from ._collections import HTTPHeaderDict
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
from .response import BaseHTTPResponse
__all__ = ["RequestMethods"]
_TYPE_ENCODE_URL_FIELDS = typing.Union[
typing.Sequence[tuple[str, typing.Union[str, bytes]]],
typing.Mapping[str, typing.Union[str, bytes]],
]
class RequestMethods:
"""
Convenience mixin for classes who implement a :meth:`urlopen` method, such
as :class:`urllib3.HTTPConnectionPool` and
:class:`urllib3.PoolManager`.
Provides behavior for making common types of HTTP request methods and
decides which type of request field encoding to use.
Specifically,
:meth:`.request_encode_url` is for sending requests whose fields are
encoded in the URL (such as GET, HEAD, DELETE).
:meth:`.request_encode_body` is for sending requests whose fields are
encoded in the *body* of the request using multipart or www-form-urlencoded
(such as for POST, PUT, PATCH).
:meth:`.request` is for making any kind of request, it will look up the
appropriate encoding format and use one of the above two methods to make
the request.
Initializer parameters:
:param headers:
Headers to include with all requests, unless other headers are given
explicitly.
"""
_encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"}
def __init__(self, headers: typing.Mapping[str, str] | None = None) -> None:
self.headers = headers or {}
def urlopen(
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
encode_multipart: bool = True,
multipart_boundary: str | None = None,
**kw: typing.Any,
) -> BaseHTTPResponse: # Abstract
raise NotImplementedError(
"Classes extending RequestMethods must implement "
"their own ``urlopen`` method."
)
def request(
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
fields: _TYPE_FIELDS | None = None,
headers: typing.Mapping[str, str] | None = None,
json: typing.Any | None = None,
**urlopen_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Make a request using :meth:`urlopen` with the appropriate encoding of
``fields`` based on the ``method`` used.
This is a convenience method that requires the least amount of manual
effort. It can be used in most situations, while still having the
option to drop down to more specific methods when necessary, such as
:meth:`request_encode_url`, :meth:`request_encode_body`,
or even the lowest level :meth:`urlopen`.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param fields:
Data to encode and send in the URL or request body, depending on ``method``.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param json:
Data to encode and send as JSON with UTF-encoded in the request body.
The ``"Content-Type"`` header will be set to ``"application/json"``
unless specified otherwise.
"""
method = method.upper()
if json is not None and body is not None:
raise TypeError(
"request got values for both 'body' and 'json' parameters which are mutually exclusive"
)
if json is not None:
if headers is None:
headers = self.headers
if not ("content-type" in map(str.lower, headers.keys())):
headers = HTTPHeaderDict(headers)
headers["Content-Type"] = "application/json"
body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode(
"utf-8"
)
if body is not None:
urlopen_kw["body"] = body
if method in self._encode_url_methods:
return self.request_encode_url(
method,
url,
fields=fields, # type: ignore[arg-type]
headers=headers,
**urlopen_kw,
)
else:
return self.request_encode_body(
method, url, fields=fields, headers=headers, **urlopen_kw
)
def request_encode_url(
self,
method: str,
url: str,
fields: _TYPE_ENCODE_URL_FIELDS | None = None,
headers: typing.Mapping[str, str] | None = None,
**urlopen_kw: str,
) -> BaseHTTPResponse:
"""
Make a request using :meth:`urlopen` with the ``fields`` encoded in
the url. This is useful for request methods like GET, HEAD, DELETE, etc.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param fields:
Data to encode and send in the URL.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
"""
if headers is None:
headers = self.headers
extra_kw: dict[str, typing.Any] = {"headers": headers}
extra_kw.update(urlopen_kw)
if fields:
url += "?" + urlencode(fields)
return self.urlopen(method, url, **extra_kw)
def request_encode_body(
self,
method: str,
url: str,
fields: _TYPE_FIELDS | None = None,
headers: typing.Mapping[str, str] | None = None,
encode_multipart: bool = True,
multipart_boundary: str | None = None,
**urlopen_kw: str,
) -> BaseHTTPResponse:
"""
Make a request using :meth:`urlopen` with the ``fields`` encoded in
the body. This is useful for request methods like POST, PUT, PATCH, etc.
When ``encode_multipart=True`` (default), then
:func:`urllib3.encode_multipart_formdata` is used to encode
the payload with the appropriate content type. Otherwise
:func:`urllib.parse.urlencode` is used with the
'application/x-www-form-urlencoded' content type.
Multipart encoding must be used when posting files, and it's reasonably
safe to use it in other times too. However, it may break request
signing, such as with OAuth.
Supports an optional ``fields`` parameter of key/value strings AND
key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
the MIME type is optional. For example::
fields = {
'foo': 'bar',
'fakefile': ('foofile.txt', 'contents of foofile'),
'realfile': ('barfile.txt', open('realfile').read()),
'typedfile': ('bazfile.bin', open('bazfile').read(),
'image/jpeg'),
'nonamefile': 'contents of nonamefile field',
}
When uploading a file, providing a filename (the first parameter of the
tuple) is optional but recommended to best mimic behavior of browsers.
Note that if ``headers`` are supplied, the 'Content-Type' header will
be overwritten because it depends on the dynamic random boundary string
which is used to compose the body of the request. The random boundary
string can be explicitly set with the ``multipart_boundary`` parameter.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param fields:
Data to encode and send in the request body.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param encode_multipart:
If True, encode the ``fields`` using the multipart/form-data MIME
format.
:param multipart_boundary:
If not specified, then a random boundary will be generated using
:func:`urllib3.filepost.choose_boundary`.
"""
if headers is None:
headers = self.headers
extra_kw: dict[str, typing.Any] = {"headers": HTTPHeaderDict(headers)}
body: bytes | str
if fields:
if "body" in urlopen_kw:
raise TypeError(
"request got values for both 'fields' and 'body', can only specify one."
)
if encode_multipart:
body, content_type = encode_multipart_formdata(
fields, boundary=multipart_boundary
)
else:
body, content_type = (
urlencode(fields), # type: ignore[arg-type]
"application/x-www-form-urlencoded",
)
extra_kw["body"] = body
extra_kw["headers"].setdefault("Content-Type", content_type)
extra_kw.update(urlopen_kw)
return self.urlopen(method, url, **extra_kw)
@@ -1,24 +1,2 @@
# file generated by vcs-versioning
# don't change, don't track in version control
from __future__ import annotations
__all__ = [
"__version__",
"__version_tuple__",
"version",
"version_tuple",
"__commit_id__",
"commit_id",
]
version: str
__version__: str
__version_tuple__: tuple[int | str, ...]
version_tuple: tuple[int | str, ...]
commit_id: str | None
__commit_id__: str | None
__version__ = version = '2.7.0'
__version_tuple__ = version_tuple = (2, 7, 0)
__commit_id__ = commit_id = None
# This file is protected via CODEOWNERS
__version__ = "1.26.20"
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,17 +0,0 @@
from __future__ import annotations
import urllib3.connection
from ...connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from .connection import EmscriptenHTTPConnection, EmscriptenHTTPSConnection
def inject_into_urllib3() -> None:
# override connection classes to use emscripten specific classes
# n.b. mypy complains about the overriding of classes below
# if it isn't ignored
HTTPConnectionPool.ConnectionCls = EmscriptenHTTPConnection
HTTPSConnectionPool.ConnectionCls = EmscriptenHTTPSConnection
urllib3.connection.HTTPConnection = EmscriptenHTTPConnection # type: ignore[misc,assignment]
urllib3.connection.HTTPSConnection = EmscriptenHTTPSConnection # type: ignore[misc,assignment]
urllib3.connection.VerifiedHTTPSConnection = EmscriptenHTTPSConnection # type: ignore[assignment]
@@ -1,260 +0,0 @@
from __future__ import annotations
import os
import typing
# use http.client.HTTPException for consistency with non-emscripten
from http.client import HTTPException as HTTPException # noqa: F401
from http.client import ResponseNotReady
from ..._base_connection import _TYPE_BODY
from ...connection import HTTPConnection, ProxyConfig, port_by_scheme
from ...exceptions import TimeoutError
from ...response import BaseHTTPResponse
from ...util.connection import _TYPE_SOCKET_OPTIONS
from ...util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
from ...util.url import Url
from .fetch import _RequestError, _TimeoutError, send_request, send_streaming_request
from .request import EmscriptenRequest
from .response import EmscriptenHttpResponseWrapper, EmscriptenResponse
if typing.TYPE_CHECKING:
from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
class EmscriptenHTTPConnection:
default_port: typing.ClassVar[int] = port_by_scheme["http"]
default_socket_options: typing.ClassVar[_TYPE_SOCKET_OPTIONS]
timeout: None | (float)
host: str
port: int
blocksize: int
source_address: tuple[str, int] | None
socket_options: _TYPE_SOCKET_OPTIONS | None
proxy: Url | None
proxy_config: ProxyConfig | None
is_verified: bool = False
proxy_is_verified: bool | None = None
response_class: type[BaseHTTPResponse] = EmscriptenHttpResponseWrapper
_response: EmscriptenResponse | None
def __init__(
self,
host: str,
port: int = 0,
*,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
blocksize: int = 8192,
socket_options: _TYPE_SOCKET_OPTIONS | None = None,
proxy: Url | None = None,
proxy_config: ProxyConfig | None = None,
) -> None:
self.host = host
self.port = port
self.timeout = timeout if isinstance(timeout, float) else 0.0
self.scheme = "http"
self._closed = True
self._response = None
# ignore these things because we don't
# have control over that stuff
self.proxy = None
self.proxy_config = None
self.blocksize = blocksize
self.source_address = None
self.socket_options = None
self.is_verified = False
def set_tunnel(
self,
host: str,
port: int | None = 0,
headers: typing.Mapping[str, str] | None = None,
scheme: str = "http",
) -> None:
pass
def connect(self) -> None:
pass
def request(
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
# We know *at least* botocore is depending on the order of the
# first 3 parameters so to be safe we only mark the later ones
# as keyword-only to ensure we have space to extend.
*,
chunked: bool = False,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> None:
self._closed = False
if url.startswith("/"):
if self.port is not None:
port = f":{self.port}"
else:
port = ""
# no scheme / host / port included, make a full url
url = f"{self.scheme}://{self.host}{port}{url}"
request = EmscriptenRequest(
url=url,
method=method,
timeout=self.timeout if self.timeout else 0,
decode_content=decode_content,
)
request.set_body(body)
if headers:
for k, v in headers.items():
request.set_header(k, v)
self._response = None
try:
if not preload_content:
self._response = send_streaming_request(request)
if self._response is None:
self._response = send_request(request)
except _TimeoutError as e:
raise TimeoutError(e.message) from e
except _RequestError as e:
raise HTTPException(e.message) from e
def getresponse(self) -> BaseHTTPResponse:
if self._response is not None:
return EmscriptenHttpResponseWrapper(
internal_response=self._response,
url=self._response.request.url,
connection=self,
)
else:
raise ResponseNotReady()
def close(self) -> None:
self._closed = True
self._response = None
@property
def is_closed(self) -> bool:
"""Whether the connection either is brand new or has been previously closed.
If this property is True then both ``is_connected`` and ``has_connected_to_proxy``
properties must be False.
"""
return self._closed
@property
def is_connected(self) -> bool:
"""Whether the connection is actively connected to any origin (proxy or target)"""
return True
@property
def has_connected_to_proxy(self) -> bool:
"""Whether the connection has successfully connected to its proxy.
This returns False if no proxy is in use. Used to determine whether
errors are coming from the proxy layer or from tunnelling to the target origin.
"""
return False
class EmscriptenHTTPSConnection(EmscriptenHTTPConnection):
default_port = port_by_scheme["https"]
# all this is basically ignored, as browser handles https
cert_reqs: int | str | None = None
ca_certs: str | None = None
ca_cert_dir: str | None = None
ca_cert_data: None | str | bytes = None
cert_file: str | None
key_file: str | None
key_password: str | None
ssl_context: typing.Any | None
ssl_version: int | str | None = None
ssl_minimum_version: int | None = None
ssl_maximum_version: int | None = None
assert_hostname: None | str | typing.Literal[False]
assert_fingerprint: str | None = None
def __init__(
self,
host: str,
port: int = 0,
*,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
blocksize: int = 16384,
socket_options: (
None | _TYPE_SOCKET_OPTIONS
) = HTTPConnection.default_socket_options,
proxy: Url | None = None,
proxy_config: ProxyConfig | None = None,
cert_reqs: int | str | None = None,
assert_hostname: None | str | typing.Literal[False] = None,
assert_fingerprint: str | None = None,
server_hostname: str | None = None,
ssl_context: typing.Any | None = None,
ca_certs: str | None = None,
ca_cert_dir: str | None = None,
ca_cert_data: None | str | bytes = None,
ssl_minimum_version: int | None = None,
ssl_maximum_version: int | None = None,
ssl_version: int | str | None = None, # Deprecated
cert_file: str | None = None,
key_file: str | None = None,
key_password: str | None = None,
) -> None:
super().__init__(
host,
port=port,
timeout=timeout,
source_address=source_address,
blocksize=blocksize,
socket_options=socket_options,
proxy=proxy,
proxy_config=proxy_config,
)
self.scheme = "https"
self.key_file = key_file
self.cert_file = cert_file
self.key_password = key_password
self.ssl_context = ssl_context
self.server_hostname = server_hostname
self.assert_hostname = assert_hostname
self.assert_fingerprint = assert_fingerprint
self.ssl_version = ssl_version
self.ssl_minimum_version = ssl_minimum_version
self.ssl_maximum_version = ssl_maximum_version
self.ca_certs = ca_certs and os.path.expanduser(ca_certs)
self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir)
self.ca_cert_data = ca_cert_data
self.cert_reqs = None
# The browser will automatically verify all requests.
# We have no control over that setting.
self.is_verified = True
def set_cert(
self,
key_file: str | None = None,
cert_file: str | None = None,
cert_reqs: int | str | None = None,
key_password: str | None = None,
ca_certs: str | None = None,
assert_hostname: None | str | typing.Literal[False] = None,
assert_fingerprint: str | None = None,
ca_cert_dir: str | None = None,
ca_cert_data: None | str | bytes = None,
) -> None:
pass
# verify that this class implements BaseHTTP(s) connection correctly
if typing.TYPE_CHECKING:
_supports_http_protocol: BaseHTTPConnection = EmscriptenHTTPConnection("", 0)
_supports_https_protocol: BaseHTTPSConnection = EmscriptenHTTPSConnection("", 0)
@@ -1,110 +0,0 @@
let Status = {
SUCCESS_HEADER: -1,
SUCCESS_EOF: -2,
ERROR_TIMEOUT: -3,
ERROR_EXCEPTION: -4,
};
let connections = new Map();
let nextConnectionID = 1;
const encoder = new TextEncoder();
self.addEventListener("message", async function (event) {
if (event.data.close) {
let connectionID = event.data.close;
connections.delete(connectionID);
return;
} else if (event.data.getMore) {
let connectionID = event.data.getMore;
let { curOffset, value, reader, intBuffer, byteBuffer } =
connections.get(connectionID);
// if we still have some in buffer, then just send it back straight away
if (!value || curOffset >= value.length) {
// read another buffer if required
try {
let readResponse = await reader.read();
if (readResponse.done) {
// read everything - clear connection and return
connections.delete(connectionID);
Atomics.store(intBuffer, 0, Status.SUCCESS_EOF);
Atomics.notify(intBuffer, 0);
// finished reading successfully
// return from event handler
return;
}
curOffset = 0;
connections.get(connectionID).value = readResponse.value;
value = readResponse.value;
} catch (error) {
console.log("Request exception:", error);
let errorBytes = encoder.encode(error.message);
let written = errorBytes.length;
byteBuffer.set(errorBytes);
intBuffer[1] = written;
Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION);
Atomics.notify(intBuffer, 0);
}
}
// send as much buffer as we can
let curLen = value.length - curOffset;
if (curLen > byteBuffer.length) {
curLen = byteBuffer.length;
}
byteBuffer.set(value.subarray(curOffset, curOffset + curLen), 0);
Atomics.store(intBuffer, 0, curLen); // store current length in bytes
Atomics.notify(intBuffer, 0);
curOffset += curLen;
connections.get(connectionID).curOffset = curOffset;
return;
} else {
// start fetch
let connectionID = nextConnectionID;
nextConnectionID += 1;
const intBuffer = new Int32Array(event.data.buffer);
const byteBuffer = new Uint8Array(event.data.buffer, 8);
try {
const response = await fetch(event.data.url, event.data.fetchParams);
// return the headers first via textencoder
var headers = [];
for (const pair of response.headers.entries()) {
headers.push([pair[0], pair[1]]);
}
let headerObj = {
headers: headers,
status: response.status,
connectionID,
};
const headerText = JSON.stringify(headerObj);
let headerBytes = encoder.encode(headerText);
let written = headerBytes.length;
byteBuffer.set(headerBytes);
intBuffer[1] = written;
// make a connection
connections.set(connectionID, {
reader: response.body.getReader(),
intBuffer: intBuffer,
byteBuffer: byteBuffer,
value: undefined,
curOffset: 0,
});
// set header ready
Atomics.store(intBuffer, 0, Status.SUCCESS_HEADER);
Atomics.notify(intBuffer, 0);
// all fetching after this goes through a new postmessage call with getMore
// this allows for parallel requests
} catch (error) {
console.log("Request exception:", error);
let errorBytes = encoder.encode(error.message);
let written = errorBytes.length;
byteBuffer.set(errorBytes);
intBuffer[1] = written;
Atomics.store(intBuffer, 0, Status.ERROR_EXCEPTION);
Atomics.notify(intBuffer, 0);
}
}
});
self.postMessage({ inited: true });
@@ -1,726 +0,0 @@
"""
Support for streaming http requests in emscripten.
A few caveats -
If your browser (or Node.js) has WebAssembly JavaScript Promise Integration enabled
https://github.com/WebAssembly/js-promise-integration/blob/main/proposals/js-promise-integration/Overview.md
*and* you launch pyodide using `pyodide.runPythonAsync`, this will fetch data using the
JavaScript asynchronous fetch api (wrapped via `pyodide.ffi.call_sync`). In this case
timeouts and streaming should just work.
Otherwise, it uses a combination of XMLHttpRequest and a web-worker for streaming.
This approach has several caveats:
Firstly, you can't do streaming http in the main UI thread, because atomics.wait isn't allowed.
Streaming only works if you're running pyodide in a web worker.
Secondly, this uses an extra web worker and SharedArrayBuffer to do the asynchronous fetch
operation, so it requires that you have crossOriginIsolation enabled, by serving over https
(or from localhost) with the two headers below set:
Cross-Origin-Opener-Policy: same-origin
Cross-Origin-Embedder-Policy: require-corp
You can tell if cross origin isolation is successfully enabled by looking at the global crossOriginIsolated variable in
JavaScript console. If it isn't, streaming requests will fallback to XMLHttpRequest, i.e. getting the whole
request into a buffer and then returning it. it shows a warning in the JavaScript console in this case.
Finally, the webworker which does the streaming fetch is created on initial import, but will only be started once
control is returned to javascript. Call `await wait_for_streaming_ready()` to wait for streaming fetch.
NB: in this code, there are a lot of JavaScript objects. They are named js_*
to make it clear what type of object they are.
"""
from __future__ import annotations
import io
import json
from email.parser import Parser
from importlib.resources import files
from typing import TYPE_CHECKING, Any
import js # type: ignore[import-not-found]
from pyodide.ffi import ( # type: ignore[import-not-found]
JsArray,
JsException,
JsProxy,
to_js,
)
if TYPE_CHECKING:
from typing_extensions import Buffer
from .request import EmscriptenRequest
from .response import EmscriptenResponse
"""
There are some headers that trigger unintended CORS preflight requests.
See also https://github.com/koenvo/pyodide-http/issues/22
"""
HEADERS_TO_IGNORE = ("user-agent",)
SUCCESS_HEADER = -1
SUCCESS_EOF = -2
ERROR_TIMEOUT = -3
ERROR_EXCEPTION = -4
class _RequestError(Exception):
def __init__(
self,
message: str | None = None,
*,
request: EmscriptenRequest | None = None,
response: EmscriptenResponse | None = None,
):
self.request = request
self.response = response
self.message = message
super().__init__(self.message)
class _StreamingError(_RequestError):
pass
class _TimeoutError(_RequestError):
pass
def _obj_from_dict(dict_val: dict[str, Any]) -> JsProxy:
return to_js(dict_val, dict_converter=js.Object.fromEntries)
class _ReadStream(io.RawIOBase):
def __init__(
self,
int_buffer: JsArray,
byte_buffer: JsArray,
timeout: float,
worker: JsProxy,
connection_id: int,
request: EmscriptenRequest,
):
self.int_buffer = int_buffer
self.byte_buffer = byte_buffer
self.read_pos = 0
self.read_len = 0
self.connection_id = connection_id
self.worker = worker
self.timeout = int(1000 * timeout) if timeout > 0 else None
self.is_live = True
self._is_closed = False
self.request: EmscriptenRequest | None = request
def __del__(self) -> None:
self.close()
# this is compatible with _base_connection
def is_closed(self) -> bool:
return self._is_closed
# for compatibility with RawIOBase
@property
def closed(self) -> bool:
return self.is_closed()
def close(self) -> None:
if self.is_closed():
return
self.read_len = 0
self.read_pos = 0
self.int_buffer = None
self.byte_buffer = None
self._is_closed = True
self.request = None
if self.is_live:
self.worker.postMessage(_obj_from_dict({"close": self.connection_id}))
self.is_live = False
super().close()
def readable(self) -> bool:
return True
def writable(self) -> bool:
return False
def seekable(self) -> bool:
return False
def readinto(self, byte_obj: Buffer) -> int:
if not self.int_buffer:
raise _StreamingError(
"No buffer for stream in _ReadStream.readinto",
request=self.request,
response=None,
)
if self.read_len == 0:
# wait for the worker to send something
js.Atomics.store(self.int_buffer, 0, ERROR_TIMEOUT)
self.worker.postMessage(_obj_from_dict({"getMore": self.connection_id}))
if (
js.Atomics.wait(self.int_buffer, 0, ERROR_TIMEOUT, self.timeout)
== "timed-out"
):
raise _TimeoutError
data_len = self.int_buffer[0]
if data_len > 0:
self.read_len = data_len
self.read_pos = 0
elif data_len == ERROR_EXCEPTION:
string_len = self.int_buffer[1]
# decode the error string
js_decoder = js.TextDecoder.new()
json_str = js_decoder.decode(self.byte_buffer.slice(0, string_len))
raise _StreamingError(
f"Exception thrown in fetch: {json_str}",
request=self.request,
response=None,
)
else:
# EOF, free the buffers and return zero
# and free the request
self.is_live = False
self.close()
return 0
# copy from int32array to python bytes
ret_length = min(self.read_len, len(memoryview(byte_obj)))
subarray = self.byte_buffer.subarray(
self.read_pos, self.read_pos + ret_length
).to_py()
memoryview(byte_obj)[0:ret_length] = subarray
self.read_len -= ret_length
self.read_pos += ret_length
return ret_length
class _StreamingFetcher:
def __init__(self) -> None:
# make web-worker and data buffer on startup
self.streaming_ready = False
streaming_worker_code = (
files(__package__)
.joinpath("emscripten_fetch_worker.js")
.read_text(encoding="utf-8")
)
js_data_blob = js.Blob.new(
to_js([streaming_worker_code], create_pyproxies=False),
_obj_from_dict({"type": "application/javascript"}),
)
def promise_resolver(js_resolve_fn: JsProxy, js_reject_fn: JsProxy) -> None:
def onMsg(e: JsProxy) -> None:
self.streaming_ready = True
js_resolve_fn(e)
def onErr(e: JsProxy) -> None:
js_reject_fn(e) # Defensive: never happens in ci
self.js_worker.onmessage = onMsg
self.js_worker.onerror = onErr
js_data_url = js.URL.createObjectURL(js_data_blob)
self.js_worker = js.globalThis.Worker.new(js_data_url)
self.js_worker_ready_promise = js.globalThis.Promise.new(promise_resolver)
def send(self, request: EmscriptenRequest) -> EmscriptenResponse:
headers = {
k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE
}
body = request.body
fetch_data = {"headers": headers, "body": to_js(body), "method": request.method}
# start the request off in the worker
timeout = int(1000 * request.timeout) if request.timeout > 0 else None
js_shared_buffer = js.SharedArrayBuffer.new(1048576)
js_int_buffer = js.Int32Array.new(js_shared_buffer)
js_byte_buffer = js.Uint8Array.new(js_shared_buffer, 8)
js.Atomics.store(js_int_buffer, 0, ERROR_TIMEOUT)
js.Atomics.notify(js_int_buffer, 0)
js_absolute_url = js.URL.new(request.url, js.location).href
self.js_worker.postMessage(
_obj_from_dict(
{
"buffer": js_shared_buffer,
"url": js_absolute_url,
"fetchParams": fetch_data,
}
)
)
# wait for the worker to send something
js.Atomics.wait(js_int_buffer, 0, ERROR_TIMEOUT, timeout)
if js_int_buffer[0] == ERROR_TIMEOUT:
raise _TimeoutError(
"Timeout connecting to streaming request",
request=request,
response=None,
)
elif js_int_buffer[0] == SUCCESS_HEADER:
# got response
# header length is in second int of intBuffer
string_len = js_int_buffer[1]
# decode the rest to a JSON string
js_decoder = js.TextDecoder.new()
# this does a copy (the slice) because decode can't work on shared array
# for some silly reason
json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
# get it as an object
response_obj = json.loads(json_str)
return EmscriptenResponse(
request=request,
status_code=response_obj["status"],
headers=response_obj["headers"],
body=_ReadStream(
js_int_buffer,
js_byte_buffer,
request.timeout,
self.js_worker,
response_obj["connectionID"],
request,
),
)
elif js_int_buffer[0] == ERROR_EXCEPTION:
string_len = js_int_buffer[1]
# decode the error string
js_decoder = js.TextDecoder.new()
json_str = js_decoder.decode(js_byte_buffer.slice(0, string_len))
raise _StreamingError(
f"Exception thrown in fetch: {json_str}", request=request, response=None
)
else:
raise _StreamingError(
f"Unknown status from worker in fetch: {js_int_buffer[0]}",
request=request,
response=None,
)
class _JSPIReadStream(io.RawIOBase):
"""
A read stream that uses pyodide.ffi.run_sync to read from a JavaScript fetch
response. This requires support for WebAssembly JavaScript Promise Integration
in the containing browser, and for pyodide to be launched via runPythonAsync.
:param js_read_stream:
The JavaScript stream reader
:param timeout:
Timeout in seconds
:param request:
The request we're handling
:param response:
The response this stream relates to
:param js_abort_controller:
A JavaScript AbortController object, used for timeouts
"""
def __init__(
self,
js_read_stream: Any,
timeout: float,
request: EmscriptenRequest,
response: EmscriptenResponse,
js_abort_controller: Any, # JavaScript AbortController for timeouts
):
self.js_read_stream = js_read_stream
self.timeout = timeout
self._is_closed = False
self._is_done = False
self.request: EmscriptenRequest | None = request
self.response: EmscriptenResponse | None = response
self.current_buffer = None
self.current_buffer_pos = 0
self.js_abort_controller = js_abort_controller
def __del__(self) -> None:
self.close()
# this is compatible with _base_connection
def is_closed(self) -> bool:
return self._is_closed
# for compatibility with RawIOBase
@property
def closed(self) -> bool:
return self.is_closed()
def close(self) -> None:
if self.is_closed():
return
self.read_len = 0
self.read_pos = 0
self.js_read_stream.cancel()
self.js_read_stream = None
self._is_closed = True
self._is_done = True
self.request = None
self.response = None
super().close()
def readable(self) -> bool:
return True
def writable(self) -> bool:
return False
def seekable(self) -> bool:
return False
def _get_next_buffer(self) -> bool:
result_js = _run_sync_with_timeout(
self.js_read_stream.read(),
self.timeout,
self.js_abort_controller,
request=self.request,
response=self.response,
)
if result_js.done:
self._is_done = True
return False
else:
self.current_buffer = result_js.value.to_py()
self.current_buffer_pos = 0
return True
def readinto(self, byte_obj: Buffer) -> int:
if self.current_buffer is None:
if not self._get_next_buffer() or self.current_buffer is None:
self.close()
return 0
ret_length = min(
len(byte_obj), len(self.current_buffer) - self.current_buffer_pos
)
byte_obj[0:ret_length] = self.current_buffer[
self.current_buffer_pos : self.current_buffer_pos + ret_length
]
self.current_buffer_pos += ret_length
if self.current_buffer_pos == len(self.current_buffer):
self.current_buffer = None
return ret_length
# check if we are in a worker or not
def is_in_browser_main_thread() -> bool:
return hasattr(js, "window") and hasattr(js, "self") and js.self == js.window
def is_cross_origin_isolated() -> bool:
return hasattr(js, "crossOriginIsolated") and js.crossOriginIsolated
def is_in_node() -> bool:
return (
hasattr(js, "process")
and hasattr(js.process, "release")
and hasattr(js.process.release, "name")
and js.process.release.name == "node"
)
def is_worker_available() -> bool:
return hasattr(js, "Worker") and hasattr(js, "Blob")
_fetcher: _StreamingFetcher | None = None
if is_worker_available() and (
(is_cross_origin_isolated() and not is_in_browser_main_thread())
and (not is_in_node())
):
_fetcher = _StreamingFetcher()
else:
_fetcher = None
NODE_JSPI_ERROR = (
"urllib3 only works in Node.js with pyodide.runPythonAsync"
" and requires the flag --experimental-wasm-stack-switching in "
" versions of node <24."
)
def send_streaming_request(request: EmscriptenRequest) -> EmscriptenResponse | None:
if has_jspi():
return send_jspi_request(request, True)
elif is_in_node():
raise _RequestError(
message=NODE_JSPI_ERROR,
request=request,
response=None,
)
if _fetcher and streaming_ready():
return _fetcher.send(request)
else:
_show_streaming_warning()
return None
_SHOWN_TIMEOUT_WARNING = False
def _show_timeout_warning() -> None:
global _SHOWN_TIMEOUT_WARNING
if not _SHOWN_TIMEOUT_WARNING:
_SHOWN_TIMEOUT_WARNING = True
message = "Warning: Timeout is not available on main browser thread"
js.console.warn(message)
_SHOWN_STREAMING_WARNING = False
def _show_streaming_warning() -> None:
global _SHOWN_STREAMING_WARNING
if not _SHOWN_STREAMING_WARNING:
_SHOWN_STREAMING_WARNING = True
message = "Can't stream HTTP requests because: \n"
if not is_cross_origin_isolated():
message += " Page is not cross-origin isolated\n"
if is_in_browser_main_thread():
message += " Python is running in main browser thread\n"
if not is_worker_available():
message += " Worker or Blob classes are not available in this environment." # Defensive: this is always False in browsers that we test in
if streaming_ready() is False:
message += """ Streaming fetch worker isn't ready. If you want to be sure that streaming fetch
is working, you need to call: 'await urllib3.contrib.emscripten.fetch.wait_for_streaming_ready()`"""
from js import console
console.warn(message)
def send_request(request: EmscriptenRequest) -> EmscriptenResponse:
if has_jspi():
return send_jspi_request(request, False)
elif is_in_node():
raise _RequestError(
message=NODE_JSPI_ERROR,
request=request,
response=None,
)
try:
js_xhr = js.XMLHttpRequest.new()
if not is_in_browser_main_thread():
js_xhr.responseType = "arraybuffer"
if request.timeout:
js_xhr.timeout = int(request.timeout * 1000)
else:
js_xhr.overrideMimeType("text/plain; charset=ISO-8859-15")
if request.timeout:
# timeout isn't available on the main thread - show a warning in console
# if it is set
_show_timeout_warning()
js_xhr.open(request.method, request.url, False)
for name, value in request.headers.items():
if name.lower() not in HEADERS_TO_IGNORE:
js_xhr.setRequestHeader(name, value)
js_xhr.send(to_js(request.body))
headers = dict(Parser().parsestr(js_xhr.getAllResponseHeaders()))
if not is_in_browser_main_thread():
body = js_xhr.response.to_py().tobytes()
else:
body = js_xhr.response.encode("ISO-8859-15")
return EmscriptenResponse(
status_code=js_xhr.status, headers=headers, body=body, request=request
)
except JsException as err:
if err.name == "TimeoutError":
raise _TimeoutError(err.message, request=request)
elif err.name == "NetworkError":
raise _RequestError(err.message, request=request)
else:
# general http error
raise _RequestError(err.message, request=request)
def send_jspi_request(
request: EmscriptenRequest, streaming: bool
) -> EmscriptenResponse:
"""
Send a request using WebAssembly JavaScript Promise Integration
to wrap the asynchronous JavaScript fetch api (experimental).
:param request:
Request to send
:param streaming:
Whether to stream the response
:return: The response object
:rtype: EmscriptenResponse
"""
timeout = request.timeout
js_abort_controller = js.AbortController.new()
headers = {k: v for k, v in request.headers.items() if k not in HEADERS_TO_IGNORE}
req_body = request.body
fetch_data = {
"headers": headers,
"body": to_js(req_body),
"method": request.method,
"signal": js_abort_controller.signal,
}
# Node.js returns the whole response (unlike opaqueredirect in browsers),
# so urllib3 can set `redirect: manual` to control redirects itself.
# https://stackoverflow.com/a/78524615
if _is_node_js():
fetch_data["redirect"] = "manual"
# Call JavaScript fetch (async api, returns a promise)
fetcher_promise_js = js.fetch(request.url, _obj_from_dict(fetch_data))
# Now suspend WebAssembly until we resolve that promise
# or time out.
response_js = _run_sync_with_timeout(
fetcher_promise_js,
timeout,
js_abort_controller,
request=request,
response=None,
)
headers = {}
header_iter = response_js.headers.entries()
while True:
iter_value_js = header_iter.next()
if getattr(iter_value_js, "done", False):
break
else:
headers[str(iter_value_js.value[0])] = str(iter_value_js.value[1])
status_code = response_js.status
body: bytes | io.RawIOBase = b""
response = EmscriptenResponse(
status_code=status_code, headers=headers, body=b"", request=request
)
if streaming:
# get via inputstream
if response_js.body is not None:
# get a reader from the fetch response
body_stream_js = response_js.body.getReader()
body = _JSPIReadStream(
body_stream_js, timeout, request, response, js_abort_controller
)
else:
# get directly via arraybuffer
# n.b. this is another async JavaScript call.
body = _run_sync_with_timeout(
response_js.arrayBuffer(),
timeout,
js_abort_controller,
request=request,
response=response,
).to_py()
response.body = body
return response
def _run_sync_with_timeout(
promise: Any,
timeout: float,
js_abort_controller: Any,
request: EmscriptenRequest | None,
response: EmscriptenResponse | None,
) -> Any:
"""
Await a JavaScript promise synchronously with a timeout which is implemented
via the AbortController
:param promise:
Javascript promise to await
:param timeout:
Timeout in seconds
:param js_abort_controller:
A JavaScript AbortController object, used on timeout
:param request:
The request being handled
:param response:
The response being handled (if it exists yet)
:raises _TimeoutError: If the request times out
:raises _RequestError: If the request raises a JavaScript exception
:return: The result of awaiting the promise.
"""
timer_id = None
if timeout > 0:
timer_id = js.setTimeout(
js_abort_controller.abort.bind(js_abort_controller), int(timeout * 1000)
)
try:
from pyodide.ffi import run_sync
# run_sync here uses WebAssembly JavaScript Promise Integration to
# suspend python until the JavaScript promise resolves.
return run_sync(promise)
except JsException as err:
if err.name == "AbortError":
raise _TimeoutError(
message="Request timed out", request=request, response=response
)
else:
raise _RequestError(message=err.message, request=request, response=response)
finally:
if timer_id is not None:
js.clearTimeout(timer_id)
def has_jspi() -> bool:
"""
Return true if jspi can be used.
This requires both browser support and also WebAssembly
to be in the correct state - i.e. that the javascript
call into python was async not sync.
:return: True if jspi can be used.
:rtype: bool
"""
try:
from pyodide.ffi import can_run_sync, run_sync # noqa: F401
return bool(can_run_sync())
except ImportError:
return False
def _is_node_js() -> bool:
"""
Check if we are in Node.js.
:return: True if we are in Node.js.
:rtype: bool
"""
return (
hasattr(js, "process")
and hasattr(js.process, "release")
# According to the Node.js documentation, the release name is always "node".
and js.process.release.name == "node"
)
def streaming_ready() -> bool | None:
if _fetcher:
return _fetcher.streaming_ready
else:
return None # no fetcher, return None to signify that
async def wait_for_streaming_ready() -> bool:
if _fetcher:
await _fetcher.js_worker_ready_promise
return True
else:
return False
@@ -1,22 +0,0 @@
from __future__ import annotations
from dataclasses import dataclass, field
from ..._base_connection import _TYPE_BODY
@dataclass
class EmscriptenRequest:
method: str
url: str
params: dict[str, str] | None = None
body: _TYPE_BODY | None = None
headers: dict[str, str] = field(default_factory=dict)
timeout: float = 0
decode_content: bool = True
def set_header(self, name: str, value: str) -> None:
self.headers[name.capitalize()] = value
def set_body(self, body: _TYPE_BODY | None) -> None:
self.body = body
@@ -1,281 +0,0 @@
from __future__ import annotations
import json as _json
import logging
import typing
from contextlib import contextmanager
from dataclasses import dataclass
from http.client import HTTPException as HTTPException
from io import BytesIO, IOBase
from ...exceptions import InvalidHeader, TimeoutError
from ...response import BaseHTTPResponse
from ...util.retry import Retry
from .request import EmscriptenRequest
if typing.TYPE_CHECKING:
from ..._base_connection import BaseHTTPConnection, BaseHTTPSConnection
log = logging.getLogger(__name__)
@dataclass
class EmscriptenResponse:
status_code: int
headers: dict[str, str]
body: IOBase | bytes
request: EmscriptenRequest
class EmscriptenHttpResponseWrapper(BaseHTTPResponse):
def __init__(
self,
internal_response: EmscriptenResponse,
url: str | None = None,
connection: BaseHTTPConnection | BaseHTTPSConnection | None = None,
):
self._pool = None # set by pool class
self._body = None
self._uncached_read_occurred = False
self._response = internal_response
self._url = url
self._connection = connection
self._closed = False
super().__init__(
headers=internal_response.headers,
status=internal_response.status_code,
request_url=url,
version=0,
version_string="HTTP/?",
reason="",
decode_content=True,
)
self.length_remaining = self._init_length(self._response.request.method)
self.length_is_certain = False
@property
def url(self) -> str | None:
return self._url
@url.setter
def url(self, url: str | None) -> None:
self._url = url
@property
def connection(self) -> BaseHTTPConnection | BaseHTTPSConnection | None:
return self._connection
@property
def retries(self) -> Retry | None:
return self._retries
@retries.setter
def retries(self, retries: Retry | None) -> None:
# Override the request_url if retries has a redirect location.
self._retries = retries
def stream(
self, amt: int | None = 2**16, decode_content: bool | None = None
) -> typing.Generator[bytes]:
"""
A generator wrapper for the read() method. A call will block until
``amt`` bytes have been read from the connection or until the
connection is closed.
:param amt:
How much of the content to read. The generator will return up to
much data per iteration, but may return less. This is particularly
likely when using compressed data. However, the empty string will
never be returned.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
"""
while True:
data = self.read(amt=amt, decode_content=decode_content)
if data:
yield data
else:
break
def _init_length(self, request_method: str | None) -> int | None:
length: int | None
content_length: str | None = self.headers.get("content-length")
if content_length is not None:
try:
# RFC 7230 section 3.3.2 specifies multiple content lengths can
# be sent in a single Content-Length header
# (e.g. Content-Length: 42, 42). This line ensures the values
# are all valid ints and that as long as the `set` length is 1,
# all values are the same. Otherwise, the header is invalid.
lengths = {int(val) for val in content_length.split(",")}
if len(lengths) > 1:
raise InvalidHeader(
"Content-Length contained multiple "
"unmatching values (%s)" % content_length
)
length = lengths.pop()
except ValueError:
length = None
else:
if length < 0:
length = None
else: # if content_length is None
length = None
# Check for responses that shouldn't include a body
if (
self.status in (204, 304)
or 100 <= self.status < 200
or request_method == "HEAD"
):
length = 0
return length
def read(
self,
amt: int | None = None,
decode_content: bool | None = None, # ignored because browser decodes always
cache_content: bool = False,
) -> bytes:
if (
self._closed
or self._response is None
or (isinstance(self._response.body, IOBase) and self._response.body.closed)
):
return b""
with self._error_catcher():
# body has been preloaded as a string by XmlHttpRequest
if not isinstance(self._response.body, IOBase):
self.length_remaining = len(self._response.body)
self.length_is_certain = True
# wrap body in IOStream
self._response.body = BytesIO(self._response.body)
if amt is not None and amt >= 0:
# don't cache partial content
cache_content = False
data = self._response.body.read(amt)
self._uncached_read_occurred = True
else: # read all we can (and cache it)
data = self._response.body.read()
if cache_content and not self._uncached_read_occurred:
self._body = data
else:
self._uncached_read_occurred = True
if self.length_remaining is not None:
self.length_remaining = max(self.length_remaining - len(data), 0)
if len(data) == 0 or (
self.length_is_certain and self.length_remaining == 0
):
# definitely finished reading, close response stream
self._response.body.close()
return typing.cast(bytes, data)
def read_chunked(
self,
amt: int | None = None,
decode_content: bool | None = None,
) -> typing.Generator[bytes]:
# chunked is handled by browser
while True:
bytes = self.read(amt, decode_content)
if not bytes:
break
yield bytes
def release_conn(self) -> None:
if not self._pool or not self._connection:
return None
self._pool._put_conn(self._connection)
self._connection = None
def drain_conn(self) -> None:
self.close()
@property
def data(self) -> bytes:
if self._body:
return self._body
else:
return self.read(cache_content=True)
def json(self) -> typing.Any:
"""
Deserializes the body of the HTTP response as a Python object.
The body of the HTTP response must be encoded using UTF-8, as per
`RFC 8529 Section 8.1 <https://www.rfc-editor.org/rfc/rfc8259#section-8.1>`_.
To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to
your custom decoder instead.
If the body of the HTTP response is not decodable to UTF-8, a
`UnicodeDecodeError` will be raised. If the body of the HTTP response is not a
valid JSON document, a `json.JSONDecodeError` will be raised.
Read more :ref:`here <json_content>`.
:returns: The body of the HTTP response as a Python object.
"""
data = self.data.decode("utf-8")
return _json.loads(data)
def close(self) -> None:
if not self._closed:
if isinstance(self._response.body, IOBase):
self._response.body.close()
if self._connection:
self._connection.close()
self._connection = None
self._closed = True
@contextmanager
def _error_catcher(self) -> typing.Generator[None]:
"""
Catch Emscripten specific exceptions thrown by fetch.py,
instead re-raising urllib3 variants, so that low-level exceptions
are not leaked in the high-level api.
On exit, release the connection back to the pool.
"""
from .fetch import _RequestError, _TimeoutError # avoid circular import
clean_exit = False
try:
yield
# If no exception is thrown, we should avoid cleaning up
# unnecessarily.
clean_exit = True
except _TimeoutError as e:
raise TimeoutError(str(e))
except _RequestError as e:
raise HTTPException(str(e))
finally:
# If we didn't terminate cleanly, we need to throw away our
# connection.
if not clean_exit:
# The response may not be closed but we're not going to use it
# anymore so close it now
if (
isinstance(self._response.body, IOBase)
and not self._response.body.closed
):
self._response.body.close()
# release the connection back to the pool
self.release_conn()
else:
# If we have read everything from the response stream,
# return the connection back to the pool.
if (
isinstance(self._response.body, IOBase)
and self._response.body.closed
):
self.release_conn()
@@ -1,17 +1,17 @@
"""
Module for using pyOpenSSL as a TLS backend. This module was relevant before
the standard library ``ssl`` module supported SNI, but now that we've dropped
support for Python 2.7 all relevant Python versions support SNI so
**this module is no longer recommended**.
TLS with SNI_-support for Python 2. Follow these instructions if you would
like to verify TLS certificates in Python 2. Note, the default libraries do
*not* do certificate checking; you need to do additional work to validate
certificates yourself.
This needs the following packages installed:
* `pyOpenSSL`_ (tested with 19.0.0)
* `cryptography`_ (minimum 2.3, from pyopenssl)
* `idna`_ (minimum 2.1, from cryptography)
* `pyOpenSSL`_ (tested with 16.0.0)
* `cryptography`_ (minimum 1.3.4, from pyopenssl)
* `idna`_ (minimum 2.0, from cryptography)
However, pyOpenSSL depends on cryptography, so while we use all three directly here we
end up having relatively few packages required.
However, pyopenssl depends on cryptography, which depends on idna, so while we
use all three directly here we end up having relatively few packages required.
You can install them with the following command:
@@ -33,45 +33,75 @@ like this:
except ImportError:
pass
Now you can use :mod:`urllib3` as you normally would, and it will support SNI
when the required modules are installed.
Activating this module also has the positive side effect of disabling SSL/TLS
compression in Python 2 (see `CRIME attack`_).
.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication
.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit)
.. _pyopenssl: https://www.pyopenssl.org
.. _cryptography: https://cryptography.io
.. _idna: https://github.com/kjd/idna
"""
from __future__ import absolute_import
from __future__ import annotations
import OpenSSL.SSL # type: ignore[import-not-found]
import OpenSSL.crypto
import OpenSSL.SSL
from cryptography import x509
from cryptography.hazmat.backends.openssl import backend as openssl_backend
try:
from cryptography.x509 import UnsupportedExtension # type: ignore[attr-defined]
from cryptography.x509 import UnsupportedExtension
except ImportError:
# UnsupportedExtension is gone in cryptography >= 2.1.0
class UnsupportedExtension(Exception): # type: ignore[no-redef]
class UnsupportedExtension(Exception):
pass
from io import BytesIO
from socket import error as SocketError
from socket import timeout
try: # Platform-specific: Python 2
from socket import _fileobject
except ImportError: # Platform-specific: Python 3
_fileobject = None
from ..packages.backports.makefile import backport_makefile
import logging
import ssl
import typing
from io import BytesIO
from socket import socket as socket_cls
import sys
import warnings
from .. import util
from ..packages import six
from ..util.ssl_ import PROTOCOL_TLS_CLIENT
if typing.TYPE_CHECKING:
from OpenSSL.crypto import X509 # type: ignore[import-not-found]
warnings.warn(
"'urllib3.contrib.pyopenssl' module is deprecated and will be removed "
"in a future release of urllib3 2.x. Read more in this issue: "
"https://github.com/urllib3/urllib3/issues/2680",
category=DeprecationWarning,
stacklevel=2,
)
__all__ = ["inject_into_urllib3", "extract_from_urllib3"]
# SNI always works.
HAS_SNI = True
# Map from urllib3 to PyOpenSSL compatible parameter-values.
_openssl_versions: dict[int, int] = {
util.ssl_.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
util.ssl_.PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, # type: ignore[attr-defined]
_openssl_versions = {
util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD,
PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD,
ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD,
}
if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"):
_openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD
if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"):
_openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD
@@ -85,77 +115,43 @@ _stdlib_to_openssl_verify = {
ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER
+ OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
}
_openssl_to_stdlib_verify = {v: k for k, v in _stdlib_to_openssl_verify.items()}
# The SSLvX values are the most likely to be missing in the future
# but we check them all just to be sure.
_OP_NO_SSLv2_OR_SSLv3: int = getattr(OpenSSL.SSL, "OP_NO_SSLv2", 0) | getattr(
OpenSSL.SSL, "OP_NO_SSLv3", 0
)
_OP_NO_TLSv1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1", 0)
_OP_NO_TLSv1_1: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_1", 0)
_OP_NO_TLSv1_2: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_2", 0)
_OP_NO_TLSv1_3: int = getattr(OpenSSL.SSL, "OP_NO_TLSv1_3", 0)
_openssl_to_ssl_minimum_version: dict[int, int] = {
ssl.TLSVersion.MINIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
ssl.TLSVersion.TLSv1: _OP_NO_SSLv2_OR_SSLv3,
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1,
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1,
ssl.TLSVersion.TLSv1_3: (
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
),
ssl.TLSVersion.MAXIMUM_SUPPORTED: (
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2
),
}
_openssl_to_ssl_maximum_version: dict[int, int] = {
ssl.TLSVersion.MINIMUM_SUPPORTED: (
_OP_NO_SSLv2_OR_SSLv3
| _OP_NO_TLSv1
| _OP_NO_TLSv1_1
| _OP_NO_TLSv1_2
| _OP_NO_TLSv1_3
),
ssl.TLSVersion.TLSv1: (
_OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_1 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3
),
ssl.TLSVersion.TLSv1_1: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_2 | _OP_NO_TLSv1_3,
ssl.TLSVersion.TLSv1_2: _OP_NO_SSLv2_OR_SSLv3 | _OP_NO_TLSv1_3,
ssl.TLSVersion.TLSv1_3: _OP_NO_SSLv2_OR_SSLv3,
ssl.TLSVersion.MAXIMUM_SUPPORTED: _OP_NO_SSLv2_OR_SSLv3,
}
_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items())
# OpenSSL will only write 16K at a time
SSL_WRITE_BLOCKSIZE = 16384
orig_util_HAS_SNI = util.HAS_SNI
orig_util_SSLContext = util.ssl_.SSLContext
log = logging.getLogger(__name__)
def inject_into_urllib3() -> None:
def inject_into_urllib3():
"Monkey-patch urllib3 with PyOpenSSL-backed SSL-support."
_validate_dependencies_met()
util.SSLContext = PyOpenSSLContext # type: ignore[assignment]
util.ssl_.SSLContext = PyOpenSSLContext # type: ignore[assignment]
util.SSLContext = PyOpenSSLContext
util.ssl_.SSLContext = PyOpenSSLContext
util.HAS_SNI = HAS_SNI
util.ssl_.HAS_SNI = HAS_SNI
util.IS_PYOPENSSL = True
util.ssl_.IS_PYOPENSSL = True
def extract_from_urllib3() -> None:
def extract_from_urllib3():
"Undo monkey-patching by :func:`inject_into_urllib3`."
util.SSLContext = orig_util_SSLContext
util.ssl_.SSLContext = orig_util_SSLContext
util.HAS_SNI = orig_util_HAS_SNI
util.ssl_.HAS_SNI = orig_util_HAS_SNI
util.IS_PYOPENSSL = False
util.ssl_.IS_PYOPENSSL = False
def _validate_dependencies_met() -> None:
def _validate_dependencies_met():
"""
Verifies that PyOpenSSL's package-level dependencies have been met.
Throws `ImportError` if they are not met.
@@ -181,7 +177,7 @@ def _validate_dependencies_met() -> None:
)
def _dnsname_to_stdlib(name: str) -> str | None:
def _dnsname_to_stdlib(name):
"""
Converts a dNSName SubjectAlternativeName field to the form used by the
standard library on the given Python version.
@@ -195,7 +191,7 @@ def _dnsname_to_stdlib(name: str) -> str | None:
the name given should be skipped.
"""
def idna_encode(name: str) -> bytes | None:
def idna_encode(name):
"""
Borrowed wholesale from the Python Cryptography Project. It turns out
that we can't just safely call `idna.encode`: it can explode for
@@ -204,7 +200,7 @@ def _dnsname_to_stdlib(name: str) -> str | None:
import idna
try:
for prefix in ["*.", "."]:
for prefix in [u"*.", u"."]:
if name.startswith(prefix):
name = name[len(prefix) :]
return prefix.encode("ascii") + idna.encode(name)
@@ -216,17 +212,24 @@ def _dnsname_to_stdlib(name: str) -> str | None:
if ":" in name:
return name
encoded_name = idna_encode(name)
if encoded_name is None:
name = idna_encode(name)
if name is None:
return None
return encoded_name.decode("utf-8")
elif sys.version_info >= (3, 0):
name = name.decode("utf-8")
return name
def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]:
def get_subj_alt_name(peer_cert):
"""
Given an PyOpenSSL certificate, provides all the subject alternative names.
"""
cert = peer_cert.to_cryptography()
# Pass the cert to cryptography, which has much better APIs for this.
if hasattr(peer_cert, "to_cryptography"):
cert = peer_cert.to_cryptography()
else:
der = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, peer_cert)
cert = x509.load_der_x509_certificate(der, openssl_backend)
# We want to find the SAN extension. Ask Cryptography to locate it (it's
# faster than looping in Python)
@@ -270,94 +273,93 @@ def get_subj_alt_name(peer_cert: X509) -> list[tuple[str, str]]:
return names
class WrappedSocket:
"""API-compatibility wrapper for Python OpenSSL's Connection-class."""
class WrappedSocket(object):
"""API-compatibility wrapper for Python OpenSSL's Connection-class.
def __init__(
self,
connection: OpenSSL.SSL.Connection,
socket: socket_cls,
suppress_ragged_eofs: bool = True,
) -> None:
Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
collector of pypy.
"""
def __init__(self, connection, socket, suppress_ragged_eofs=True):
self.connection = connection
self.socket = socket
self.suppress_ragged_eofs = suppress_ragged_eofs
self._io_refs = 0
self._makefile_refs = 0
self._closed = False
def fileno(self) -> int:
def fileno(self):
return self.socket.fileno()
# Copy-pasted from Python 3.5 source code
def _decref_socketios(self) -> None:
if self._io_refs > 0:
self._io_refs -= 1
def _decref_socketios(self):
if self._makefile_refs > 0:
self._makefile_refs -= 1
if self._closed:
self.close()
def recv(self, *args: typing.Any, **kwargs: typing.Any) -> bytes:
def recv(self, *args, **kwargs):
try:
data = self.connection.recv(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return b""
else:
raise OSError(e.args[0], str(e)) from e
raise SocketError(str(e))
except OpenSSL.SSL.ZeroReturnError:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return b""
else:
raise
except OpenSSL.SSL.WantReadError as e:
except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise TimeoutError("The read operation timed out") from e
raise timeout("The read operation timed out")
else:
return self.recv(*args, **kwargs)
# TLS 1.3 post-handshake authentication
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"read error: {e!r}") from e
raise ssl.SSLError("read error: %r" % e)
else:
return data # type: ignore[no-any-return]
return data
def recv_into(self, *args: typing.Any, **kwargs: typing.Any) -> int:
def recv_into(self, *args, **kwargs):
try:
return self.connection.recv_into(*args, **kwargs) # type: ignore[no-any-return]
return self.connection.recv_into(*args, **kwargs)
except OpenSSL.SSL.SysCallError as e:
if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"):
return 0
else:
raise OSError(e.args[0], str(e)) from e
raise SocketError(str(e))
except OpenSSL.SSL.ZeroReturnError:
if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN:
return 0
else:
raise
except OpenSSL.SSL.WantReadError as e:
except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(self.socket, self.socket.gettimeout()):
raise TimeoutError("The read operation timed out") from e
raise timeout("The read operation timed out")
else:
return self.recv_into(*args, **kwargs)
# TLS 1.3 post-handshake authentication
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"read error: {e!r}") from e
raise ssl.SSLError("read error: %r" % e)
def settimeout(self, timeout: float) -> None:
def settimeout(self, timeout):
return self.socket.settimeout(timeout)
def _send_until_done(self, data: bytes) -> int:
def _send_until_done(self, data):
while True:
try:
return self.connection.send(data) # type: ignore[no-any-return]
except OpenSSL.SSL.WantWriteError as e:
return self.connection.send(data)
except OpenSSL.SSL.WantWriteError:
if not util.wait_for_write(self.socket, self.socket.gettimeout()):
raise TimeoutError() from e
raise timeout()
continue
except OpenSSL.SSL.SysCallError as e:
raise OSError(e.args[0], str(e)) from e
raise SocketError(str(e))
def sendall(self, data: bytes) -> None:
def sendall(self, data):
total_sent = 0
while total_sent < len(data):
sent = self._send_until_done(
@@ -365,151 +367,135 @@ class WrappedSocket:
)
total_sent += sent
def shutdown(self, how: int) -> None:
try:
self.connection.shutdown()
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"shutdown error: {e!r}") from e
def shutdown(self):
# FIXME rethrow compatible exceptions should we ever use this
self.connection.shutdown()
def close(self) -> None:
self._closed = True
if self._io_refs <= 0:
self._real_close()
def close(self):
if self._makefile_refs < 1:
try:
self._closed = True
return self.connection.close()
except OpenSSL.SSL.Error:
return
else:
self._makefile_refs -= 1
def _real_close(self) -> None:
try:
return self.connection.close() # type: ignore[no-any-return]
except OpenSSL.SSL.Error:
return
def getpeercert(
self, binary_form: bool = False
) -> dict[str, list[typing.Any]] | None:
def getpeercert(self, binary_form=False):
x509 = self.connection.get_peer_certificate()
if not x509:
return x509 # type: ignore[no-any-return]
return x509
if binary_form:
return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) # type: ignore[no-any-return]
return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509)
return {
"subject": ((("commonName", x509.get_subject().CN),),), # type: ignore[dict-item]
"subject": ((("commonName", x509.get_subject().CN),),),
"subjectAltName": get_subj_alt_name(x509),
}
def version(self) -> str:
return self.connection.get_protocol_version_name() # type: ignore[no-any-return]
def version(self):
return self.connection.get_protocol_version_name()
def selected_alpn_protocol(self) -> str | None:
alpn_proto = self.connection.get_alpn_proto_negotiated()
return alpn_proto.decode() if alpn_proto else None
def _reuse(self):
self._makefile_refs += 1
def _drop(self):
if self._makefile_refs < 1:
self.close()
else:
self._makefile_refs -= 1
WrappedSocket.makefile = socket_cls.makefile # type: ignore[attr-defined]
if _fileobject: # Platform-specific: Python 2
def makefile(self, mode, bufsize=-1):
self._makefile_refs += 1
return _fileobject(self, mode, bufsize, close=True)
else: # Platform-specific: Python 3
makefile = backport_makefile
WrappedSocket.makefile = makefile
class PyOpenSSLContext:
class PyOpenSSLContext(object):
"""
I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible
for translating the interface of the standard library ``SSLContext`` object
to calls into PyOpenSSL.
"""
def __init__(self, protocol: int) -> None:
def __init__(self, protocol):
self.protocol = _openssl_versions[protocol]
self._ctx = OpenSSL.SSL.Context(self.protocol)
self._options = 0
self.check_hostname = False
self._minimum_version: int = ssl.TLSVersion.MINIMUM_SUPPORTED
self._maximum_version: int = ssl.TLSVersion.MAXIMUM_SUPPORTED
self._verify_flags: int = ssl.VERIFY_X509_TRUSTED_FIRST
@property
def options(self) -> int:
def options(self):
return self._options
@options.setter
def options(self, value: int) -> None:
def options(self, value):
self._options = value
self._set_ctx_options()
self._ctx.set_options(value)
@property
def verify_flags(self) -> int:
return self._verify_flags
@verify_flags.setter
def verify_flags(self, value: int) -> None:
self._verify_flags = value
self._ctx.get_cert_store().set_flags(self._verify_flags)
@property
def verify_mode(self) -> int:
def verify_mode(self):
return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()]
@verify_mode.setter
def verify_mode(self, value: ssl.VerifyMode) -> None:
def verify_mode(self, value):
self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback)
def set_default_verify_paths(self) -> None:
def set_default_verify_paths(self):
self._ctx.set_default_verify_paths()
def set_ciphers(self, ciphers: bytes | str) -> None:
if isinstance(ciphers, str):
def set_ciphers(self, ciphers):
if isinstance(ciphers, six.text_type):
ciphers = ciphers.encode("utf-8")
self._ctx.set_cipher_list(ciphers)
def load_verify_locations(
self,
cafile: str | None = None,
capath: str | None = None,
cadata: bytes | None = None,
) -> None:
def load_verify_locations(self, cafile=None, capath=None, cadata=None):
if cafile is not None:
cafile = cafile.encode("utf-8") # type: ignore[assignment]
cafile = cafile.encode("utf-8")
if capath is not None:
capath = capath.encode("utf-8") # type: ignore[assignment]
capath = capath.encode("utf-8")
try:
self._ctx.load_verify_locations(cafile, capath)
if cadata is not None:
self._ctx.load_verify_locations(BytesIO(cadata))
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"unable to load trusted certificates: {e!r}") from e
raise ssl.SSLError("unable to load trusted certificates: %r" % e)
def load_cert_chain(
self,
certfile: str,
keyfile: str | None = None,
password: str | None = None,
) -> None:
try:
self._ctx.use_certificate_chain_file(certfile)
if password is not None:
if not isinstance(password, bytes):
password = password.encode("utf-8") # type: ignore[assignment]
self._ctx.set_passwd_cb(lambda *_: password)
self._ctx.use_privatekey_file(keyfile or certfile)
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"Unable to load certificate chain: {e!r}") from e
def load_cert_chain(self, certfile, keyfile=None, password=None):
self._ctx.use_certificate_chain_file(certfile)
if password is not None:
if not isinstance(password, six.binary_type):
password = password.encode("utf-8")
self._ctx.set_passwd_cb(lambda *_: password)
self._ctx.use_privatekey_file(keyfile or certfile)
def set_alpn_protocols(self, protocols: list[bytes | str]) -> None:
protocols = [util.util.to_bytes(p, "ascii") for p in protocols]
return self._ctx.set_alpn_protos(protocols) # type: ignore[no-any-return]
def set_alpn_protocols(self, protocols):
protocols = [six.ensure_binary(p) for p in protocols]
return self._ctx.set_alpn_protos(protocols)
def wrap_socket(
self,
sock: socket_cls,
server_side: bool = False,
do_handshake_on_connect: bool = True,
suppress_ragged_eofs: bool = True,
server_hostname: bytes | str | None = None,
) -> WrappedSocket:
sock,
server_side=False,
do_handshake_on_connect=True,
suppress_ragged_eofs=True,
server_hostname=None,
):
cnx = OpenSSL.SSL.Connection(self._ctx, sock)
# If server_hostname is an IP, don't use it for SNI, per RFC6066 Section 3
if server_hostname and not util.ssl_.is_ipaddress(server_hostname):
if isinstance(server_hostname, str):
server_hostname = server_hostname.encode("utf-8")
if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3
server_hostname = server_hostname.encode("utf-8")
if server_hostname is not None:
cnx.set_tlsext_host_name(server_hostname)
cnx.set_connect_state()
@@ -517,47 +503,16 @@ class PyOpenSSLContext:
while True:
try:
cnx.do_handshake()
except OpenSSL.SSL.WantReadError as e:
except OpenSSL.SSL.WantReadError:
if not util.wait_for_read(sock, sock.gettimeout()):
raise TimeoutError("select timed out") from e
raise timeout("select timed out")
continue
except OpenSSL.SSL.Error as e:
raise ssl.SSLError(f"bad handshake: {e!r}") from e
raise ssl.SSLError("bad handshake: %r" % e)
break
return WrappedSocket(cnx, sock)
def _set_ctx_options(self) -> None:
self._ctx.set_options(
self._options
| _openssl_to_ssl_minimum_version[self._minimum_version]
| _openssl_to_ssl_maximum_version[self._maximum_version]
)
@property
def minimum_version(self) -> int:
return self._minimum_version
@minimum_version.setter
def minimum_version(self, minimum_version: int) -> None:
self._minimum_version = minimum_version
self._set_ctx_options()
@property
def maximum_version(self) -> int:
return self._maximum_version
@maximum_version.setter
def maximum_version(self, maximum_version: int) -> None:
self._maximum_version = maximum_version
self._set_ctx_options()
def _verify_callback(
cnx: OpenSSL.SSL.Connection,
x509: X509,
err_no: int,
err_depth: int,
return_code: int,
) -> bool:
def _verify_callback(cnx, x509, err_no, err_depth, return_code):
return err_no == 0
@@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
"""
This module contains provisional support for SOCKS proxies from within
urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and
@@ -37,11 +38,10 @@ with the proxy:
proxy_url="socks5h://<username>:<password>@proxy-host"
"""
from __future__ import annotations
from __future__ import absolute_import
try:
import socks # type: ignore[import-untyped]
import socks
except ImportError:
import warnings
@@ -51,13 +51,13 @@ except ImportError:
(
"SOCKS support in urllib3 requires the installation of optional "
"dependencies: specifically, PySocks. For more information, see "
"https://urllib3.readthedocs.io/en/latest/advanced-usage.html#socks-proxies"
"https://urllib3.readthedocs.io/en/1.26.x/contrib.html#socks-proxies"
),
DependencyWarning,
)
raise
import typing
from socket import error as SocketError
from socket import timeout as SocketTimeout
from ..connection import HTTPConnection, HTTPSConnection
@@ -69,16 +69,7 @@ from ..util.url import parse_url
try:
import ssl
except ImportError:
ssl = None # type: ignore[assignment]
class _TYPE_SOCKS_OPTIONS(typing.TypedDict):
socks_version: int
proxy_host: str | None
proxy_port: str | None
username: str | None
password: str | None
rdns: bool
ssl = None
class SOCKSConnection(HTTPConnection):
@@ -86,20 +77,15 @@ class SOCKSConnection(HTTPConnection):
A plain-text HTTP connection that connects via a SOCKS proxy.
"""
def __init__(
self,
_socks_options: _TYPE_SOCKS_OPTIONS,
*args: typing.Any,
**kwargs: typing.Any,
) -> None:
self._socks_options = _socks_options
super().__init__(*args, **kwargs)
def __init__(self, *args, **kwargs):
self._socks_options = kwargs.pop("_socks_options")
super(SOCKSConnection, self).__init__(*args, **kwargs)
def _new_conn(self) -> socks.socksocket:
def _new_conn(self):
"""
Establish a new connection via the SOCKS proxy.
"""
extra_kw: dict[str, typing.Any] = {}
extra_kw = {}
if self.source_address:
extra_kw["source_address"] = self.source_address
@@ -116,14 +102,15 @@ class SOCKSConnection(HTTPConnection):
proxy_password=self._socks_options["password"],
proxy_rdns=self._socks_options["rdns"],
timeout=self.timeout,
**extra_kw,
**extra_kw
)
except SocketTimeout as e:
except SocketTimeout:
raise ConnectTimeoutError(
self,
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
) from e
"Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
)
except socks.ProxyError as e:
# This is fragile as hell, but it seems to be the only way to raise
@@ -133,23 +120,22 @@ class SOCKSConnection(HTTPConnection):
if isinstance(error, SocketTimeout):
raise ConnectTimeoutError(
self,
f"Connection to {self.host} timed out. (connect timeout={self.timeout})",
) from e
else:
# Adding `from e` messes with coverage somehow, so it's omitted.
# See #2386.
raise NewConnectionError(
self, f"Failed to establish a new connection: {error}"
"Connection to %s timed out. (connect timeout=%s)"
% (self.host, self.timeout),
)
else: # Defensive: see https://github.com/urllib3/urllib3/pull/3728#pullrequestreview-3816302703
else:
raise NewConnectionError(
self, "Failed to establish a new connection: %s" % error
)
else:
raise NewConnectionError(
self, f"Failed to establish a new connection: {e}"
) from e
self, "Failed to establish a new connection: %s" % e
)
except OSError as e: # Defensive: PySocks should catch all these.
except SocketError as e: # Defensive: PySocks should catch all these.
raise NewConnectionError(
self, f"Failed to establish a new connection: {e}"
) from e
self, "Failed to establish a new connection: %s" % e
)
return conn
@@ -183,12 +169,12 @@ class SOCKSProxyManager(PoolManager):
def __init__(
self,
proxy_url: str,
username: str | None = None,
password: str | None = None,
num_pools: int = 10,
headers: typing.Mapping[str, str] | None = None,
**connection_pool_kw: typing.Any,
proxy_url,
username=None,
password=None,
num_pools=10,
headers=None,
**connection_pool_kw
):
parsed = parse_url(proxy_url)
@@ -209,7 +195,7 @@ class SOCKSProxyManager(PoolManager):
socks_version = socks.PROXY_TYPE_SOCKS4
rdns = True
else:
raise ValueError(f"Unable to determine SOCKS version from {proxy_url}")
raise ValueError("Unable to determine SOCKS version from %s" % proxy_url)
self.proxy_url = proxy_url
@@ -223,6 +209,8 @@ class SOCKSProxyManager(PoolManager):
}
connection_pool_kw["_socks_options"] = socks_options
super().__init__(num_pools, headers, **connection_pool_kw)
super(SOCKSProxyManager, self).__init__(
num_pools, headers, **connection_pool_kw
)
self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme
@@ -1,16 +1,6 @@
from __future__ import annotations
from __future__ import absolute_import
import socket
import typing
import warnings
from email.errors import MessageDefect
from http.client import IncompleteRead as httplib_IncompleteRead
if typing.TYPE_CHECKING:
from .connection import HTTPConnection
from .connectionpool import ConnectionPool
from .response import HTTPResponse
from .util.retry import Retry
from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead
# Base Exceptions
@@ -18,61 +8,64 @@ if typing.TYPE_CHECKING:
class HTTPError(Exception):
"""Base exception used by this module."""
pass
class HTTPWarning(Warning):
"""Base warning used by this module."""
_TYPE_REDUCE_RESULT = tuple[typing.Callable[..., object], tuple[object, ...]]
pass
class PoolError(HTTPError):
"""Base exception for errors caused within a pool."""
def __init__(self, pool: ConnectionPool, message: str) -> None:
def __init__(self, pool, message):
self.pool = pool
self._message = message
super().__init__(f"{pool}: {message}")
HTTPError.__init__(self, "%s: %s" % (pool, message))
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
def __reduce__(self):
# For pickling purposes.
return self.__class__, (None, self._message)
return self.__class__, (None, None)
class RequestError(PoolError):
"""Base exception for PoolErrors that have associated URLs."""
def __init__(self, pool: ConnectionPool, url: str | None, message: str) -> None:
def __init__(self, pool, url, message):
self.url = url
super().__init__(pool, message)
PoolError.__init__(self, pool, message)
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
def __reduce__(self):
# For pickling purposes.
return self.__class__, (None, self.url, self._message)
return self.__class__, (None, self.url, None)
class SSLError(HTTPError):
"""Raised when SSL certificate fails in an HTTPS connection."""
pass
class ProxyError(HTTPError):
"""Raised when the connection to a proxy fails."""
# The original error is also available as __cause__.
original_error: Exception
def __init__(self, message: str, error: Exception) -> None:
super().__init__(message, error)
def __init__(self, message, error, *args):
super(ProxyError, self).__init__(message, error, *args)
self.original_error = error
class DecodeError(HTTPError):
"""Raised when automatic decoding based on Content-Type fails."""
pass
class ProtocolError(HTTPError):
"""Raised when something unexpected happens mid-request/response."""
pass
#: Renamed to ProtocolError but aliased for backwards compatibility.
ConnectionError = ProtocolError
@@ -86,40 +79,33 @@ class MaxRetryError(RequestError):
:param pool: The connection pool
:type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
:param str url: The requested Url
:param reason: The underlying error
:type reason: :class:`Exception`
:param string url: The requested Url
:param exceptions.Exception reason: The underlying error
"""
def __init__(
self, pool: ConnectionPool, url: str | None, reason: Exception | None = None
) -> None:
def __init__(self, pool, url, reason=None):
self.reason = reason
message = f"Max retries exceeded with url: {url} (Caused by {reason!r})"
message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason)
super().__init__(pool, url, message)
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
# For pickling purposes.
return self.__class__, (None, self.url, self.reason)
RequestError.__init__(self, pool, url, message)
class HostChangedError(RequestError):
"""Raised when an existing pool gets a request for a foreign host."""
def __init__(
self, pool: ConnectionPool, url: str, retries: Retry | int = 3
) -> None:
message = f"Tried to open a foreign host with url: {url}"
super().__init__(pool, url, message)
def __init__(self, pool, url, retries=3):
message = "Tried to open a foreign host with url: %s" % url
RequestError.__init__(self, pool, url, message)
self.retries = retries
class TimeoutStateError(HTTPError):
"""Raised when passing an invalid state to a timeout"""
pass
class TimeoutError(HTTPError):
"""Raised when a socket timeout error occurs.
@@ -128,77 +114,53 @@ class TimeoutError(HTTPError):
<ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`.
"""
pass
class ReadTimeoutError(TimeoutError, RequestError):
"""Raised when a socket timeout occurs while receiving data from a server"""
pass
# This timeout error does not have a URL attached and needs to inherit from the
# base HTTPError
class ConnectTimeoutError(TimeoutError):
"""Raised when a socket timeout occurs while connecting to a server"""
pass
class NewConnectionError(ConnectTimeoutError, HTTPError):
class NewConnectionError(ConnectTimeoutError, PoolError):
"""Raised when we fail to establish a new connection. Usually ECONNREFUSED."""
def __init__(self, conn: HTTPConnection, message: str) -> None:
self.conn = conn
self._message = message
super().__init__(f"{conn}: {message}")
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
# For pickling purposes.
return self.__class__, (None, self._message)
@property
def pool(self) -> HTTPConnection:
warnings.warn(
"The 'pool' property is deprecated and will be removed "
"in urllib3 v3.0. Use 'conn' instead.",
FutureWarning,
stacklevel=2,
)
return self.conn
class NameResolutionError(NewConnectionError):
"""Raised when host name resolution fails."""
def __init__(self, host: str, conn: HTTPConnection, reason: socket.gaierror):
message = f"Failed to resolve '{host}' ({reason})"
self._host = host
self._reason = reason
super().__init__(conn, message)
def __reduce__(self) -> _TYPE_REDUCE_RESULT:
# For pickling purposes.
return self.__class__, (self._host, None, self._reason)
pass
class EmptyPoolError(PoolError):
"""Raised when a pool runs out of connections and no more are allowed."""
class FullPoolError(PoolError):
"""Raised when we try to add a connection to a full pool in blocking mode."""
pass
class ClosedPoolError(PoolError):
"""Raised when a request enters a pool after the pool has been closed."""
pass
class LocationValueError(ValueError, HTTPError):
"""Raised when there is something wrong with a given URL input."""
pass
class LocationParseError(LocationValueError):
"""Raised when get_host or similar fails to parse the URL input."""
def __init__(self, location: str) -> None:
message = f"Failed to parse: {location}"
super().__init__(message)
def __init__(self, location):
message = "Failed to parse: %s" % location
HTTPError.__init__(self, message)
self.location = location
@@ -206,9 +168,9 @@ class LocationParseError(LocationValueError):
class URLSchemeUnknown(LocationValueError):
"""Raised when a URL input has an unsupported scheme."""
def __init__(self, scheme: str):
message = f"Not supported URL scheme {scheme}"
super().__init__(message)
def __init__(self, scheme):
message = "Not supported URL scheme %s" % scheme
super(URLSchemeUnknown, self).__init__(message)
self.scheme = scheme
@@ -223,22 +185,38 @@ class ResponseError(HTTPError):
class SecurityWarning(HTTPWarning):
"""Warned when performing security reducing actions"""
pass
class SubjectAltNameWarning(SecurityWarning):
"""Warned when connecting to a host with a certificate missing a SAN."""
pass
class InsecureRequestWarning(SecurityWarning):
"""Warned when making an unverified HTTPS request."""
class NotOpenSSLWarning(SecurityWarning):
"""Warned when using unsupported SSL library"""
pass
class SystemTimeWarning(SecurityWarning):
"""Warned when system time is suspected to be wrong"""
pass
class InsecurePlatformWarning(SecurityWarning):
"""Warned when certain TLS/SSL configuration is not available on a platform."""
pass
class SNIMissingWarning(HTTPWarning):
"""Warned when making a HTTPS request without SNI available."""
pass
class DependencyWarning(HTTPWarning):
"""
@@ -246,10 +224,14 @@ class DependencyWarning(HTTPWarning):
dependencies.
"""
pass
class ResponseNotChunked(ProtocolError, ValueError):
"""Response needs to be chunked in order to read it as chunks."""
pass
class BodyNotHttplibCompatible(HTTPError):
"""
@@ -257,6 +239,8 @@ class BodyNotHttplibCompatible(HTTPError):
(have an fp attribute which returns raw chunks) for read_chunked().
"""
pass
class IncompleteRead(HTTPError, httplib_IncompleteRead):
"""
@@ -266,14 +250,10 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead):
for ``partial`` to avoid creating large objects on streamed reads.
"""
partial: int # type: ignore[assignment]
expected: int
def __init__(self, partial, expected):
super(IncompleteRead, self).__init__(partial, expected)
def __init__(self, partial: int, expected: int) -> None:
self.partial = partial
self.expected = expected
def __repr__(self) -> str:
def __repr__(self):
return "IncompleteRead(%i bytes read, %i more expected)" % (
self.partial,
self.expected,
@@ -283,13 +263,14 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead):
class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
"""Invalid chunk length in a chunked response."""
def __init__(self, response: HTTPResponse, length: bytes) -> None:
self.partial: int = response.tell() # type: ignore[assignment]
self.expected: int | None = response.length_remaining
def __init__(self, response, length):
super(InvalidChunkLength, self).__init__(
response.tell(), response.length_remaining
)
self.response = response
self.length = length
def __repr__(self) -> str:
def __repr__(self):
return "InvalidChunkLength(got length %r, %i bytes read)" % (
self.length,
self.partial,
@@ -299,13 +280,15 @@ class InvalidChunkLength(HTTPError, httplib_IncompleteRead):
class InvalidHeader(HTTPError):
"""The header provided was somehow invalid."""
pass
class ProxySchemeUnknown(AssertionError, URLSchemeUnknown):
"""ProxyManager does not support the supplied scheme"""
# TODO(t-8ch): Stop inheriting from AssertionError in v2.0.
def __init__(self, scheme: str | None) -> None:
def __init__(self, scheme):
# 'localhost' is here because our URL parser parses
# localhost:8080 -> scheme=localhost, remove if we fix this.
if scheme == "localhost":
@@ -313,23 +296,28 @@ class ProxySchemeUnknown(AssertionError, URLSchemeUnknown):
if scheme is None:
message = "Proxy URL had no scheme, should start with http:// or https://"
else:
message = f"Proxy URL had unsupported scheme {scheme}, should use http:// or https://"
super().__init__(message)
message = (
"Proxy URL had unsupported scheme %s, should use http:// or https://"
% scheme
)
super(ProxySchemeUnknown, self).__init__(message)
class ProxySchemeUnsupported(ValueError):
"""Fetching HTTPS resources through HTTPS proxies is unsupported"""
pass
class HeaderParsingError(HTTPError):
"""Raised by assert_header_parsing, but we convert it to a log.warning statement."""
def __init__(
self, defects: list[MessageDefect], unparsed_data: bytes | str | None
) -> None:
message = f"{defects or 'Unknown'}, unparsed data: {unparsed_data!r}"
super().__init__(message)
def __init__(self, defects, unparsed_data):
message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data)
super(HeaderParsingError, self).__init__(message)
class UnrewindableBodyError(HTTPError):
"""urllib3 encountered an error when trying to rewind a body"""
pass
@@ -1,20 +1,13 @@
from __future__ import annotations
from __future__ import absolute_import
import email.utils
import mimetypes
import typing
import re
_TYPE_FIELD_VALUE = typing.Union[str, bytes]
_TYPE_FIELD_VALUE_TUPLE = typing.Union[
_TYPE_FIELD_VALUE,
tuple[str, _TYPE_FIELD_VALUE],
tuple[str, _TYPE_FIELD_VALUE, str],
]
from .packages import six
def guess_content_type(
filename: str | None, default: str = "application/octet-stream"
) -> str:
def guess_content_type(filename, default="application/octet-stream"):
"""
Guess the "Content-Type" of a file.
@@ -28,7 +21,7 @@ def guess_content_type(
return default
def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
def format_header_param_rfc2231(name, value):
"""
Helper function to format and quote a single header parameter using the
strategy defined in RFC 2231.
@@ -41,28 +34,14 @@ def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
The name of the parameter, a string expected to be ASCII only.
:param value:
The value of the parameter, provided as ``bytes`` or `str``.
:returns:
:ret:
An RFC-2231-formatted unicode string.
.. deprecated:: 2.0.0
Will be removed in urllib3 v3.0. This is not valid for
``multipart/form-data`` header parameters.
"""
import warnings
warnings.warn(
"'format_header_param_rfc2231' is insecure, deprecated and will be "
"removed in urllib3 v3.0. This is not valid for "
"multipart/form-data header parameters.",
FutureWarning,
stacklevel=2,
)
if isinstance(value, bytes):
if isinstance(value, six.binary_type):
value = value.decode("utf-8")
if not any(ch in value for ch in '"\\\r\n'):
result = f'{name}="{value}"'
result = u'%s="%s"' % (name, value)
try:
result.encode("ascii")
except (UnicodeEncodeError, UnicodeDecodeError):
@@ -70,87 +49,81 @@ def format_header_param_rfc2231(name: str, value: _TYPE_FIELD_VALUE) -> str:
else:
return result
if six.PY2: # Python 2:
value = value.encode("utf-8")
# encode_rfc2231 accepts an encoded string and returns an ascii-encoded
# string in Python 2 but accepts and returns unicode strings in Python 3
value = email.utils.encode_rfc2231(value, "utf-8")
value = f"{name}*={value}"
value = "%s*=%s" % (name, value)
if six.PY2: # Python 2:
value = value.decode("utf-8")
return value
def format_multipart_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
_HTML5_REPLACEMENTS = {
u"\u0022": u"%22",
# Replace "\" with "\\".
u"\u005C": u"\u005C\u005C",
}
# All control characters from 0x00 to 0x1F *except* 0x1B.
_HTML5_REPLACEMENTS.update(
{
six.unichr(cc): u"%{:02X}".format(cc)
for cc in range(0x00, 0x1F + 1)
if cc not in (0x1B,)
}
)
def _replace_multiple(value, needles_and_replacements):
def replacer(match):
return needles_and_replacements[match.group(0)]
pattern = re.compile(
r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()])
)
result = pattern.sub(replacer, value)
return result
def format_header_param_html5(name, value):
"""
Format and quote a single multipart header parameter.
Helper function to format and quote a single header parameter using the
HTML5 strategy.
This follows the `WHATWG HTML Standard`_ as of 2021/06/10, matching
the behavior of current browser and curl versions. Values are
assumed to be UTF-8. The ``\\n``, ``\\r``, and ``"`` characters are
percent encoded.
Particularly useful for header parameters which might contain
non-ASCII values, like file names. This follows the `HTML5 Working Draft
Section 4.10.22.7`_ and matches the behavior of curl and modern browsers.
.. _WHATWG HTML Standard:
https://html.spec.whatwg.org/multipage/
form-control-infrastructure.html#multipart-form-data
.. _HTML5 Working Draft Section 4.10.22.7:
https://w3c.github.io/html/sec-forms.html#multipart-form-data
:param name:
The name of the parameter, an ASCII-only ``str``.
The name of the parameter, a string expected to be ASCII only.
:param value:
The value of the parameter, a ``str`` or UTF-8 encoded
``bytes``.
:returns:
A string ``name="value"`` with the escaped value.
.. versionchanged:: 2.0.0
Matches the WHATWG HTML Standard as of 2021/06/10. Control
characters are no longer percent encoded.
.. versionchanged:: 2.0.0
Renamed from ``format_header_param_html5`` and
``format_header_param``. The old names will be removed in
urllib3 v3.0.
The value of the parameter, provided as ``bytes`` or `str``.
:ret:
A unicode string, stripped of troublesome characters.
"""
if isinstance(value, bytes):
if isinstance(value, six.binary_type):
value = value.decode("utf-8")
# percent encode \n \r "
value = value.translate({10: "%0A", 13: "%0D", 34: "%22"})
return f'{name}="{value}"'
value = _replace_multiple(value, _HTML5_REPLACEMENTS)
return u'%s="%s"' % (name, value)
def format_header_param_html5(name: str, value: _TYPE_FIELD_VALUE) -> str:
"""
.. deprecated:: 2.0.0
Renamed to :func:`format_multipart_header_param`. Will be
removed in urllib3 v3.0.
"""
import warnings
warnings.warn(
"'format_header_param_html5' has been renamed to "
"'format_multipart_header_param'. The old name will be "
"removed in urllib3 v3.0.",
FutureWarning,
stacklevel=2,
)
return format_multipart_header_param(name, value)
# For backwards-compatibility.
format_header_param = format_header_param_html5
def format_header_param(name: str, value: _TYPE_FIELD_VALUE) -> str:
"""
.. deprecated:: 2.0.0
Renamed to :func:`format_multipart_header_param`. Will be
removed in urllib3 v3.0.
"""
import warnings
warnings.warn(
"'format_header_param' has been renamed to "
"'format_multipart_header_param'. The old name will be "
"removed in urllib3 v3.0.",
FutureWarning,
stacklevel=2,
)
return format_multipart_header_param(name, value)
class RequestField:
class RequestField(object):
"""
A data container for request body parameters.
@@ -162,47 +135,29 @@ class RequestField:
An optional filename of the request field. Must be unicode.
:param headers:
An optional dict-like object of headers to initially use for the field.
.. versionchanged:: 2.0.0
The ``header_formatter`` parameter is deprecated and will
be removed in urllib3 v3.0.
:param header_formatter:
An optional callable that is used to encode and format the headers. By
default, this is :func:`format_header_param_html5`.
"""
def __init__(
self,
name: str,
data: _TYPE_FIELD_VALUE,
filename: str | None = None,
headers: typing.Mapping[str, str] | None = None,
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
name,
data,
filename=None,
headers=None,
header_formatter=format_header_param_html5,
):
self._name = name
self._filename = filename
self.data = data
self.headers: dict[str, str | None] = {}
self.headers = {}
if headers:
self.headers = dict(headers)
if header_formatter is not None:
import warnings
warnings.warn(
"The 'header_formatter' parameter is deprecated and "
"will be removed in urllib3 v3.0.",
FutureWarning,
stacklevel=2,
)
self.header_formatter = header_formatter
else:
self.header_formatter = format_multipart_header_param
self.header_formatter = header_formatter
@classmethod
def from_tuples(
cls,
fieldname: str,
value: _TYPE_FIELD_VALUE_TUPLE,
header_formatter: typing.Callable[[str, _TYPE_FIELD_VALUE], str] | None = None,
) -> RequestField:
def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5):
"""
A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
@@ -219,10 +174,6 @@ class RequestField:
Field names and filenames must be unicode.
"""
filename: str | None
content_type: str | None
data: _TYPE_FIELD_VALUE
if isinstance(value, tuple):
if len(value) == 3:
filename, data, content_type = value
@@ -241,29 +192,20 @@ class RequestField:
return request_param
def _render_part(self, name: str, value: _TYPE_FIELD_VALUE) -> str:
def _render_part(self, name, value):
"""
Override this method to change how each multipart header
parameter is formatted. By default, this calls
:func:`format_multipart_header_param`.
Overridable helper function to format a single header parameter. By
default, this calls ``self.header_formatter``.
:param name:
The name of the parameter, an ASCII-only ``str``.
The name of the parameter, a string expected to be ASCII only.
:param value:
The value of the parameter, a ``str`` or UTF-8 encoded
``bytes``.
:meta public:
The value of the parameter, provided as a unicode string.
"""
return self.header_formatter(name, value)
def _render_parts(
self,
header_parts: (
dict[str, _TYPE_FIELD_VALUE | None]
| typing.Sequence[tuple[str, _TYPE_FIELD_VALUE | None]]
),
) -> str:
def _render_parts(self, header_parts):
"""
Helper function to format and quote a single header.
@@ -274,21 +216,18 @@ class RequestField:
A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format
as `k1="v1"; k2="v2"; ...`.
"""
iterable: typing.Iterable[tuple[str, _TYPE_FIELD_VALUE | None]]
parts = []
iterable = header_parts
if isinstance(header_parts, dict):
iterable = header_parts.items()
else:
iterable = header_parts
for name, value in iterable:
if value is not None:
parts.append(self._render_part(name, value))
return "; ".join(parts)
return u"; ".join(parts)
def render_headers(self) -> str:
def render_headers(self):
"""
Renders the headers for this request field.
"""
@@ -297,45 +236,39 @@ class RequestField:
sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"]
for sort_key in sort_keys:
if self.headers.get(sort_key, False):
lines.append(f"{sort_key}: {self.headers[sort_key]}")
lines.append(u"%s: %s" % (sort_key, self.headers[sort_key]))
for header_name, header_value in self.headers.items():
if header_name not in sort_keys:
if header_value:
lines.append(f"{header_name}: {header_value}")
lines.append(u"%s: %s" % (header_name, header_value))
lines.append("\r\n")
return "\r\n".join(lines)
lines.append(u"\r\n")
return u"\r\n".join(lines)
def make_multipart(
self,
content_disposition: str | None = None,
content_type: str | None = None,
content_location: str | None = None,
) -> None:
self, content_disposition=None, content_type=None, content_location=None
):
"""
Makes this request field into a multipart request field.
This method overrides "Content-Disposition", "Content-Type" and
"Content-Location" headers to the request parameter.
:param content_disposition:
The 'Content-Disposition' of the request body. Defaults to 'form-data'
:param content_type:
The 'Content-Type' of the request body.
:param content_location:
The 'Content-Location' of the request body.
"""
content_disposition = (content_disposition or "form-data") + "; ".join(
self.headers["Content-Disposition"] = content_disposition or u"form-data"
self.headers["Content-Disposition"] += u"; ".join(
[
"",
u"",
self._render_parts(
(("name", self._name), ("filename", self._filename))
((u"name", self._name), (u"filename", self._filename))
),
]
)
self.headers["Content-Disposition"] = content_disposition
self.headers["Content-Type"] = content_type
self.headers["Content-Location"] = content_location
@@ -1,32 +1,28 @@
from __future__ import annotations
from __future__ import absolute_import
import binascii
import codecs
import os
import typing
from io import BytesIO
from .fields import _TYPE_FIELD_VALUE_TUPLE, RequestField
from .fields import RequestField
from .packages import six
from .packages.six import b
writer = codecs.lookup("utf-8")[3]
_TYPE_FIELDS_SEQUENCE = typing.Sequence[
typing.Union[tuple[str, _TYPE_FIELD_VALUE_TUPLE], RequestField]
]
_TYPE_FIELDS = typing.Union[
_TYPE_FIELDS_SEQUENCE,
typing.Mapping[str, _TYPE_FIELD_VALUE_TUPLE],
]
def choose_boundary() -> str:
def choose_boundary():
"""
Our embarrassingly-simple replacement for mimetools.choose_boundary.
"""
return binascii.hexlify(os.urandom(16)).decode()
boundary = binascii.hexlify(os.urandom(16))
if not six.PY2:
boundary = boundary.decode("ascii")
return boundary
def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]:
def iter_field_objects(fields):
"""
Iterate over fields.
@@ -34,29 +30,42 @@ def iter_field_objects(fields: _TYPE_FIELDS) -> typing.Iterable[RequestField]:
:class:`~urllib3.fields.RequestField`.
"""
iterable: typing.Iterable[RequestField | tuple[str, _TYPE_FIELD_VALUE_TUPLE]]
if isinstance(fields, typing.Mapping):
iterable = fields.items()
if isinstance(fields, dict):
i = six.iteritems(fields)
else:
iterable = fields
i = iter(fields)
for field in iterable:
for field in i:
if isinstance(field, RequestField):
yield field
else:
yield RequestField.from_tuples(*field)
def encode_multipart_formdata(
fields: _TYPE_FIELDS, boundary: str | None = None
) -> tuple[bytes, str]:
def iter_fields(fields):
"""
.. deprecated:: 1.6
Iterate over fields.
The addition of :class:`~urllib3.fields.RequestField` makes this function
obsolete. Instead, use :func:`iter_field_objects`, which returns
:class:`~urllib3.fields.RequestField` objects.
Supports list of (k, v) tuples and dicts.
"""
if isinstance(fields, dict):
return ((k, v) for k, v in six.iteritems(fields))
return ((k, v) for k, v in fields)
def encode_multipart_formdata(fields, boundary=None):
"""
Encode a dictionary of ``fields`` using the multipart/form-data MIME format.
:param fields:
Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`).
Values are processed by :func:`urllib3.fields.RequestField.from_tuples`.
:param boundary:
If not specified, then a random boundary will be generated using
@@ -67,7 +76,7 @@ def encode_multipart_formdata(
boundary = choose_boundary()
for field in iter_field_objects(fields):
body.write(f"--{boundary}\r\n".encode("latin-1"))
body.write(b("--%s\r\n" % (boundary)))
writer(body).write(field.render_headers())
data = field.data
@@ -75,15 +84,15 @@ def encode_multipart_formdata(
if isinstance(data, int):
data = str(data) # Backwards compatibility
if isinstance(data, str):
if isinstance(data, six.text_type):
writer(body).write(data)
else:
body.write(data)
body.write(b"\r\n")
body.write(f"--{boundary}--\r\n".encode("latin-1"))
body.write(b("--%s--\r\n" % (boundary)))
content_type = f"multipart/form-data; boundary={boundary}"
content_type = str("multipart/form-data; boundary=%s" % boundary)
return body.getvalue(), content_type
@@ -1,53 +0,0 @@
from __future__ import annotations
from importlib.metadata import version
__all__ = [
"inject_into_urllib3",
"extract_from_urllib3",
]
import typing
orig_HTTPSConnection: typing.Any = None
def inject_into_urllib3() -> None:
# First check if h2 version is valid
h2_version = version("h2")
if not h2_version.startswith("4."):
raise ImportError(
"urllib3 v2 supports h2 version 4.x.x, currently "
f"the 'h2' module is compiled with {h2_version!r}. "
"See: https://github.com/urllib3/urllib3/issues/3290"
)
# Import here to avoid circular dependencies.
from .. import connection as urllib3_connection
from .. import util as urllib3_util
from ..connectionpool import HTTPSConnectionPool
from ..util import ssl_ as urllib3_util_ssl
from .connection import HTTP2Connection
global orig_HTTPSConnection
orig_HTTPSConnection = urllib3_connection.HTTPSConnection
HTTPSConnectionPool.ConnectionCls = HTTP2Connection
urllib3_connection.HTTPSConnection = HTTP2Connection # type: ignore[misc]
# TODO: Offer 'http/1.1' as well, but for testing purposes this is handy.
urllib3_util.ALPN_PROTOCOLS = ["h2"]
urllib3_util_ssl.ALPN_PROTOCOLS = ["h2"]
def extract_from_urllib3() -> None:
from .. import connection as urllib3_connection
from .. import util as urllib3_util
from ..connectionpool import HTTPSConnectionPool
from ..util import ssl_ as urllib3_util_ssl
HTTPSConnectionPool.ConnectionCls = orig_HTTPSConnection
urllib3_connection.HTTPSConnection = orig_HTTPSConnection # type: ignore[misc]
urllib3_util.ALPN_PROTOCOLS = ["http/1.1"]
urllib3_util_ssl.ALPN_PROTOCOLS = ["http/1.1"]
@@ -1,356 +0,0 @@
from __future__ import annotations
import logging
import re
import threading
import types
import typing
import h2.config
import h2.connection
import h2.events
from .._base_connection import _TYPE_BODY
from .._collections import HTTPHeaderDict
from ..connection import HTTPSConnection, _get_default_user_agent
from ..exceptions import ConnectionError
from ..response import BaseHTTPResponse
orig_HTTPSConnection = HTTPSConnection
T = typing.TypeVar("T")
log = logging.getLogger(__name__)
RE_IS_LEGAL_HEADER_NAME = re.compile(rb"^[!#$%&'*+\-.^_`|~0-9a-z]+$")
RE_IS_ILLEGAL_HEADER_VALUE = re.compile(rb"[\0\x00\x0a\x0d\r\n]|^[ \r\n\t]|[ \r\n\t]$")
def _is_legal_header_name(name: bytes) -> bool:
"""
"An implementation that validates fields according to the definitions in Sections
5.1 and 5.5 of [HTTP] only needs an additional check that field names do not
include uppercase characters." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
`http.client._is_legal_header_name` does not validate the field name according to the
HTTP 1.1 spec, so we do that here, in addition to checking for uppercase characters.
This does not allow for the `:` character in the header name, so should not
be used to validate pseudo-headers.
"""
return bool(RE_IS_LEGAL_HEADER_NAME.match(name))
def _is_illegal_header_value(value: bytes) -> bool:
"""
"A field value MUST NOT contain the zero value (ASCII NUL, 0x00), line feed
(ASCII LF, 0x0a), or carriage return (ASCII CR, 0x0d) at any position. A field
value MUST NOT start or end with an ASCII whitespace character (ASCII SP or HTAB,
0x20 or 0x09)." (https://httpwg.org/specs/rfc9113.html#n-field-validity)
"""
return bool(RE_IS_ILLEGAL_HEADER_VALUE.search(value))
class _LockedObject(typing.Generic[T]):
"""
A wrapper class that hides a specific object behind a lock.
The goal here is to provide a simple way to protect access to an object
that cannot safely be simultaneously accessed from multiple threads. The
intended use of this class is simple: take hold of it with a context
manager, which returns the protected object.
"""
__slots__ = (
"lock",
"_obj",
)
def __init__(self, obj: T):
self.lock = threading.RLock()
self._obj = obj
def __enter__(self) -> T:
self.lock.acquire()
return self._obj
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: types.TracebackType | None,
) -> None:
self.lock.release()
class HTTP2Connection(HTTPSConnection):
def __init__(
self, host: str, port: int | None = None, **kwargs: typing.Any
) -> None:
self._h2_conn = self._new_h2_conn()
self._h2_stream: int | None = None
self._headers: list[tuple[bytes, bytes]] = []
if "proxy" in kwargs or "proxy_config" in kwargs: # Defensive:
raise NotImplementedError("Proxies aren't supported with HTTP/2")
super().__init__(host, port, **kwargs)
if self._tunnel_host is not None:
raise NotImplementedError("Tunneling isn't supported with HTTP/2")
def _new_h2_conn(self) -> _LockedObject[h2.connection.H2Connection]:
config = h2.config.H2Configuration(client_side=True)
return _LockedObject(h2.connection.H2Connection(config=config))
def connect(self) -> None:
super().connect()
with self._h2_conn as conn:
conn.initiate_connection()
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
def putrequest( # type: ignore[override]
self,
method: str,
url: str,
**kwargs: typing.Any,
) -> None:
"""putrequest
This deviates from the HTTPConnection method signature since we never need to override
sending accept-encoding headers or the host header.
"""
if "skip_host" in kwargs:
raise NotImplementedError("`skip_host` isn't supported")
if "skip_accept_encoding" in kwargs:
raise NotImplementedError("`skip_accept_encoding` isn't supported")
self._request_url = url or "/"
self._validate_path(url) # type: ignore[attr-defined]
if ":" in self.host:
authority = f"[{self.host}]:{self.port or 443}"
else:
authority = f"{self.host}:{self.port or 443}"
self._headers.append((b":scheme", b"https"))
self._headers.append((b":method", method.encode()))
self._headers.append((b":authority", authority.encode()))
self._headers.append((b":path", url.encode()))
with self._h2_conn as conn:
self._h2_stream = conn.get_next_available_stream_id()
def putheader(self, header: str | bytes, *values: str | bytes) -> None: # type: ignore[override]
# TODO SKIPPABLE_HEADERS from urllib3 are ignored.
header = header.encode() if isinstance(header, str) else header
header = header.lower() # A lot of upstream code uses capitalized headers.
if not _is_legal_header_name(header):
raise ValueError(f"Illegal header name {str(header)}")
for value in values:
value = value.encode() if isinstance(value, str) else value
if _is_illegal_header_value(value):
raise ValueError(f"Illegal header value {str(value)}")
self._headers.append((header, value))
def endheaders(self, message_body: typing.Any = None) -> None: # type: ignore[override]
if self._h2_stream is None:
raise ConnectionError("Must call `putrequest` first.")
with self._h2_conn as conn:
conn.send_headers(
stream_id=self._h2_stream,
headers=self._headers,
end_stream=(message_body is None),
)
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
self._headers = [] # Reset headers for the next request.
def send(self, data: typing.Any) -> None:
"""Send data to the server.
`data` can be: `str`, `bytes`, an iterable, or file-like objects
that support a .read() method.
"""
if self._h2_stream is None:
raise ConnectionError("Must call `putrequest` first.")
with self._h2_conn as conn:
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
if hasattr(data, "read"): # file-like objects
while True:
chunk = data.read(self.blocksize)
if not chunk:
break
if isinstance(chunk, str):
chunk = chunk.encode()
conn.send_data(self._h2_stream, chunk, end_stream=False)
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
conn.end_stream(self._h2_stream)
return
if isinstance(data, str): # str -> bytes
data = data.encode()
try:
if isinstance(data, bytes):
conn.send_data(self._h2_stream, data, end_stream=True)
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
else:
for chunk in data:
conn.send_data(self._h2_stream, chunk, end_stream=False)
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
conn.end_stream(self._h2_stream)
except TypeError:
raise TypeError(
"`data` should be str, bytes, iterable, or file. got %r"
% type(data)
)
def set_tunnel(
self,
host: str,
port: int | None = None,
headers: typing.Mapping[str, str] | None = None,
scheme: str = "http",
) -> None:
raise NotImplementedError(
"HTTP/2 does not support setting up a tunnel through a proxy"
)
def getresponse( # type: ignore[override]
self,
) -> HTTP2Response:
status = None
data = bytearray()
with self._h2_conn as conn:
end_stream = False
while not end_stream:
# TODO: Arbitrary read value.
if received_data := self.sock.recv(65535):
events = conn.receive_data(received_data)
for event in events:
if isinstance(event, h2.events.ResponseReceived):
headers = HTTPHeaderDict()
for header, value in event.headers:
if header == b":status":
status = int(value.decode())
else:
headers.add(
header.decode("ascii"), value.decode("ascii")
)
elif isinstance(event, h2.events.DataReceived):
data += event.data
conn.acknowledge_received_data(
event.flow_controlled_length, event.stream_id
)
elif isinstance(event, h2.events.StreamEnded):
end_stream = True
if data_to_send := conn.data_to_send():
self.sock.sendall(data_to_send)
assert status is not None
return HTTP2Response(
status=status,
headers=headers,
request_url=self._request_url,
data=bytes(data),
)
def request( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
*,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
**kwargs: typing.Any,
) -> None:
"""Send an HTTP/2 request"""
if "chunked" in kwargs:
# TODO this is often present from upstream.
# raise NotImplementedError("`chunked` isn't supported with HTTP/2")
pass
if self.sock is not None:
self.sock.settimeout(self.timeout)
self.putrequest(method, url)
headers = headers or {}
for k, v in headers.items():
if k.lower() == "transfer-encoding" and v == "chunked":
continue
else:
self.putheader(k, v)
if b"user-agent" not in dict(self._headers):
self.putheader(b"user-agent", _get_default_user_agent())
if body:
self.endheaders(message_body=body)
self.send(body)
else:
self.endheaders()
def close(self) -> None:
with self._h2_conn as conn:
try:
conn.close_connection()
if data := conn.data_to_send():
self.sock.sendall(data)
except Exception:
pass
# Reset all our HTTP/2 connection state.
self._h2_conn = self._new_h2_conn()
self._h2_stream = None
self._headers = []
super().close()
class HTTP2Response(BaseHTTPResponse):
# TODO: This is a woefully incomplete response object, but works for non-streaming.
def __init__(
self,
status: int,
headers: HTTPHeaderDict,
request_url: str,
data: bytes,
decode_content: bool = False, # TODO: support decoding
) -> None:
super().__init__(
status=status,
headers=headers,
# Following CPython, we map HTTP versions to major * 10 + minor integers
version=20,
version_string="HTTP/2",
# No reason phrase in HTTP/2
reason=None,
decode_content=decode_content,
request_url=request_url,
)
self._data = data
self.length_remaining = 0
@property
def data(self) -> bytes:
return self._data
def get_redirect_location(self) -> None:
return None
def close(self) -> None:
pass
@@ -1,87 +0,0 @@
from __future__ import annotations
import threading
class _HTTP2ProbeCache:
__slots__ = (
"_lock",
"_cache_locks",
"_cache_values",
)
def __init__(self) -> None:
self._lock = threading.Lock()
self._cache_locks: dict[tuple[str, int], threading.RLock] = {}
self._cache_values: dict[tuple[str, int], bool | None] = {}
def acquire_and_get(self, host: str, port: int) -> bool | None:
# By the end of this block we know that
# _cache_[values,locks] is available.
value = None
with self._lock:
key = (host, port)
try:
value = self._cache_values[key]
# If it's a known value we return right away.
if value is not None:
return value
except KeyError:
self._cache_locks[key] = threading.RLock()
self._cache_values[key] = None
# If the value is unknown, we acquire the lock to signal
# to the requesting thread that the probe is in progress
# or that the current thread needs to return their findings.
key_lock = self._cache_locks[key]
key_lock.acquire()
try:
# If the by the time we get the lock the value has been
# updated we want to return the updated value.
value = self._cache_values[key]
# In case an exception like KeyboardInterrupt is raised here.
except BaseException as e: # Defensive:
assert not isinstance(e, KeyError) # KeyError shouldn't be possible.
key_lock.release()
raise
return value
def set_and_release(
self, host: str, port: int, supports_http2: bool | None
) -> None:
key = (host, port)
key_lock = self._cache_locks[key]
with key_lock: # Uses an RLock, so can be locked again from same thread.
if supports_http2 is None and self._cache_values[key] is not None:
raise ValueError(
"Cannot reset HTTP/2 support for origin after value has been set."
) # Defensive: not expected in normal usage
self._cache_values[key] = supports_http2
key_lock.release()
def _values(self) -> dict[tuple[str, int], bool | None]:
"""This function is for testing purposes only. Gets the current state of the probe cache"""
with self._lock:
return {k: v for k, v in self._cache_values.items()}
def _reset(self) -> None:
"""This function is for testing purposes only. Reset the cache values"""
with self._lock:
self._cache_locks = {}
self._cache_values = {}
_HTTP2_PROBE_CACHE = _HTTP2ProbeCache()
set_and_release = _HTTP2_PROBE_CACHE.set_and_release
acquire_and_get = _HTTP2_PROBE_CACHE.acquire_and_get
_values = _HTTP2_PROBE_CACHE._values
_reset = _HTTP2_PROBE_CACHE._reset
__all__ = [
"set_and_release",
"acquire_and_get",
]
@@ -1,33 +1,24 @@
from __future__ import annotations
from __future__ import absolute_import
import collections
import functools
import logging
import typing
import warnings
from types import TracebackType
from urllib.parse import urljoin
from ._collections import HTTPHeaderDict, RecentlyUsedContainer
from ._request_methods import RequestMethods
from .connection import ProxyConfig
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme
from .exceptions import (
LocationValueError,
MaxRetryError,
ProxySchemeUnknown,
ProxySchemeUnsupported,
URLSchemeUnknown,
)
from .response import BaseHTTPResponse
from .util.connection import _TYPE_SOCKET_OPTIONS
from .packages import six
from .packages.six.moves.urllib.parse import urljoin
from .request import RequestMethods
from .util.proxy import connection_requires_http_tunnel
from .util.retry import Retry
from .util.timeout import Timeout
from .util.url import Url, parse_url
if typing.TYPE_CHECKING:
import ssl
from typing_extensions import Self
from .util.url import parse_url
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
@@ -39,62 +30,53 @@ SSL_KEYWORDS = (
"cert_file",
"cert_reqs",
"ca_certs",
"ca_cert_data",
"ssl_version",
"ssl_minimum_version",
"ssl_maximum_version",
"ca_cert_dir",
"ssl_context",
"key_password",
"server_hostname",
)
# Default value for `blocksize` - a new parameter introduced to
# http.client.HTTPConnection & http.client.HTTPSConnection in Python 3.7
_DEFAULT_BLOCKSIZE = 16384
# All known keyword arguments that could be provided to the pool manager, its
# pools, or the underlying connections. This is used to construct a pool key.
_key_fields = (
"key_scheme", # str
"key_host", # str
"key_port", # int
"key_timeout", # int or float or Timeout
"key_retries", # int or Retry
"key_strict", # bool
"key_block", # bool
"key_source_address", # str
"key_key_file", # str
"key_key_password", # str
"key_cert_file", # str
"key_cert_reqs", # str
"key_ca_certs", # str
"key_ssl_version", # str
"key_ca_cert_dir", # str
"key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext
"key_maxsize", # int
"key_headers", # dict
"key__proxy", # parsed proxy url
"key__proxy_headers", # dict
"key__proxy_config", # class
"key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples
"key__socks_options", # dict
"key_assert_hostname", # bool or string
"key_assert_fingerprint", # str
"key_server_hostname", # str
)
#: The namedtuple class used to construct keys for the connection pool.
#: All custom key schemes should include the fields in this key at a minimum.
PoolKey = collections.namedtuple("PoolKey", _key_fields)
_proxy_config_fields = ("ssl_context", "use_forwarding_for_https")
ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields)
class PoolKey(typing.NamedTuple):
"""
All known keyword arguments that could be provided to the pool manager, its
pools, or the underlying connections.
All custom key schemes should include the fields in this key at a minimum.
"""
key_scheme: str
key_host: str
key_port: int | None
key_timeout: Timeout | float | int | None
key_retries: Retry | bool | int | None
key_block: bool | None
key_source_address: tuple[str, int] | None
key_key_file: str | None
key_key_password: str | None
key_cert_file: str | None
key_cert_reqs: str | None
key_ca_certs: str | None
key_ca_cert_data: str | bytes | None
key_ssl_version: int | str | None
key_ssl_minimum_version: ssl.TLSVersion | None
key_ssl_maximum_version: ssl.TLSVersion | None
key_ca_cert_dir: str | None
key_ssl_context: ssl.SSLContext | None
key_maxsize: int | None
key_headers: frozenset[tuple[str, str]] | None
key__proxy: Url | None
key__proxy_headers: frozenset[tuple[str, str]] | None
key__proxy_config: ProxyConfig | None
key_socket_options: _TYPE_SOCKET_OPTIONS | None
key__socks_options: frozenset[tuple[str, str]] | None
key_assert_hostname: bool | str | None
key_assert_fingerprint: str | None
key_server_hostname: str | None
key_blocksize: int | None
def _default_key_normalizer(
key_class: type[PoolKey], request_context: dict[str, typing.Any]
) -> PoolKey:
def _default_key_normalizer(key_class, request_context):
"""
Create a pool key out of a request context dictionary.
@@ -140,10 +122,6 @@ def _default_key_normalizer(
if field not in context:
context[field] = None
# Default key_blocksize to _DEFAULT_BLOCKSIZE if missing from the context
if context.get("key_blocksize") is None:
context["key_blocksize"] = _DEFAULT_BLOCKSIZE
return key_class(**context)
@@ -176,50 +154,23 @@ class PoolManager(RequestMethods):
Additional parameters are used to create fresh
:class:`urllib3.connectionpool.ConnectionPool` instances.
Example:
Example::
.. code-block:: python
import urllib3
http = urllib3.PoolManager(num_pools=2)
resp1 = http.request("GET", "https://google.com/")
resp2 = http.request("GET", "https://google.com/mail")
resp3 = http.request("GET", "https://yahoo.com/")
print(len(http.pools))
# 2
>>> manager = PoolManager(num_pools=2)
>>> r = manager.request('GET', 'http://google.com/')
>>> r = manager.request('GET', 'http://google.com/mail')
>>> r = manager.request('GET', 'http://yahoo.com/')
>>> len(manager.pools)
2
"""
proxy: Url | None = None
proxy_config: ProxyConfig | None = None
proxy = None
proxy_config = None
def __init__(
self,
num_pools: int = 10,
headers: typing.Mapping[str, str] | None = None,
**connection_pool_kw: typing.Any,
) -> None:
super().__init__(headers)
# PoolManager handles redirects itself in PoolManager.urlopen().
# It always passes redirect=False to the underlying connection pool to
# suppress per-pool redirect handling. If the user supplied a non-Retry
# value (int/bool/etc) for retries and we let the pool normalize it
# while redirect=False, the resulting Retry object would have redirect
# handling disabled, which can interfere with PoolManager's own
# redirect logic. Normalize here so redirects remain governed solely by
# PoolManager logic.
if "retries" in connection_pool_kw:
retries = connection_pool_kw["retries"]
if not isinstance(retries, Retry):
retries = Retry.from_int(retries)
connection_pool_kw = connection_pool_kw.copy()
connection_pool_kw["retries"] = retries
def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
RequestMethods.__init__(self, headers)
self.connection_pool_kw = connection_pool_kw
self.pools: RecentlyUsedContainer[PoolKey, HTTPConnectionPool]
self.pools = RecentlyUsedContainer(num_pools)
# Locally set the pool classes and keys so other PoolManagers can
@@ -227,26 +178,15 @@ class PoolManager(RequestMethods):
self.pool_classes_by_scheme = pool_classes_by_scheme
self.key_fn_by_scheme = key_fn_by_scheme.copy()
def __enter__(self) -> Self:
def __enter__(self):
return self
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_val: BaseException | None,
exc_tb: TracebackType | None,
) -> typing.Literal[False]:
def __exit__(self, exc_type, exc_val, exc_tb):
self.clear()
# Return False to re-raise any potential exceptions
return False
def _new_pool(
self,
scheme: str,
host: str,
port: int,
request_context: dict[str, typing.Any] | None = None,
) -> HTTPConnectionPool:
def _new_pool(self, scheme, host, port, request_context=None):
"""
Create a new :class:`urllib3.connectionpool.ConnectionPool` based on host, port, scheme, and
any additional pool keyword arguments.
@@ -256,15 +196,10 @@ class PoolManager(RequestMethods):
connection pools handed out by :meth:`connection_from_url` and
companion methods. It is intended to be overridden for customization.
"""
pool_cls: type[HTTPConnectionPool] = self.pool_classes_by_scheme[scheme]
pool_cls = self.pool_classes_by_scheme[scheme]
if request_context is None:
request_context = self.connection_pool_kw.copy()
# Default blocksize to _DEFAULT_BLOCKSIZE if missing or explicitly
# set to 'None' in the request_context.
if request_context.get("blocksize") is None:
request_context["blocksize"] = _DEFAULT_BLOCKSIZE
# Although the context has everything necessary to create the pool,
# this function has historically only used the scheme, host, and port
# in the positional args. When an API change is acceptable these can
@@ -278,7 +213,7 @@ class PoolManager(RequestMethods):
return pool_cls(host, port, **request_context)
def clear(self) -> None:
def clear(self):
"""
Empty our store of pools and direct them all to close.
@@ -287,13 +222,7 @@ class PoolManager(RequestMethods):
"""
self.pools.clear()
def connection_from_host(
self,
host: str | None,
port: int | None = None,
scheme: str | None = "http",
pool_kwargs: dict[str, typing.Any] | None = None,
) -> HTTPConnectionPool:
def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
"""
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the host, port, and scheme.
@@ -316,23 +245,13 @@ class PoolManager(RequestMethods):
return self.connection_from_context(request_context)
def connection_from_context(
self, request_context: dict[str, typing.Any]
) -> HTTPConnectionPool:
def connection_from_context(self, request_context):
"""
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the request context.
``request_context`` must at least contain the ``scheme`` key and its
value must be a key in ``key_fn_by_scheme`` instance variable.
"""
if "strict" in request_context:
warnings.warn(
"The 'strict' parameter is no longer needed on Python 3+. "
"This will raise an error in urllib3 v3.0.",
FutureWarning,
)
request_context.pop("strict")
scheme = request_context["scheme"].lower()
pool_key_constructor = self.key_fn_by_scheme.get(scheme)
if not pool_key_constructor:
@@ -341,9 +260,7 @@ class PoolManager(RequestMethods):
return self.connection_from_pool_key(pool_key, request_context=request_context)
def connection_from_pool_key(
self, pool_key: PoolKey, request_context: dict[str, typing.Any]
) -> HTTPConnectionPool:
def connection_from_pool_key(self, pool_key, request_context=None):
"""
Get a :class:`urllib3.connectionpool.ConnectionPool` based on the provided pool key.
@@ -367,9 +284,7 @@ class PoolManager(RequestMethods):
return pool
def connection_from_url(
self, url: str, pool_kwargs: dict[str, typing.Any] | None = None
) -> HTTPConnectionPool:
def connection_from_url(self, url, pool_kwargs=None):
"""
Similar to :func:`urllib3.connectionpool.connection_from_url`.
@@ -385,9 +300,7 @@ class PoolManager(RequestMethods):
u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs
)
def _merge_pool_kwargs(
self, override: dict[str, typing.Any] | None
) -> dict[str, typing.Any]:
def _merge_pool_kwargs(self, override):
"""
Merge a dictionary of override values for self.connection_pool_kw.
@@ -407,7 +320,7 @@ class PoolManager(RequestMethods):
base_pool_kwargs[key] = value
return base_pool_kwargs
def _proxy_requires_url_absolute_form(self, parsed_url: Url) -> bool:
def _proxy_requires_url_absolute_form(self, parsed_url):
"""
Indicates if the proxy requires the complete destination URL in the
request. Normally this is only needed when not using an HTTP CONNECT
@@ -420,9 +333,24 @@ class PoolManager(RequestMethods):
self.proxy, self.proxy_config, parsed_url.scheme
)
def urlopen( # type: ignore[override]
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
) -> BaseHTTPResponse:
def _validate_proxy_scheme_url_selection(self, url_scheme):
"""
Validates that were not attempting to do TLS in TLS connections on
Python2 or with unsupported SSL implementations.
"""
if self.proxy is None or url_scheme != "https":
return
if self.proxy.scheme != "https":
return
if six.PY2 and not self.proxy_config.use_forwarding_for_https:
raise ProxySchemeUnsupported(
"Contacting HTTPS destinations through HTTPS proxies "
"'via CONNECT tunnels' is not supported in Python 2"
)
def urlopen(self, method, url, redirect=True, **kw):
"""
Same as :meth:`urllib3.HTTPConnectionPool.urlopen`
with custom cross-host redirect logic and only sends the request-uri
@@ -432,16 +360,7 @@ class PoolManager(RequestMethods):
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
u = parse_url(url)
if u.scheme is None:
warnings.warn(
"URLs without a scheme (ie 'https://') are deprecated and will raise an error "
"in urllib3 v3.0. To avoid this FutureWarning ensure all URLs "
"start with 'https://' or 'http://'. Read more in this issue: "
"https://github.com/urllib3/urllib3/issues/2920",
category=FutureWarning,
stacklevel=2,
)
self._validate_proxy_scheme_url_selection(u.scheme)
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
@@ -449,7 +368,7 @@ class PoolManager(RequestMethods):
kw["redirect"] = False
if "headers" not in kw:
kw["headers"] = self.headers
kw["headers"] = self.headers.copy()
if self._proxy_requires_url_absolute_form(u):
response = conn.urlopen(method, url, **kw)
@@ -470,7 +389,7 @@ class PoolManager(RequestMethods):
kw["body"] = None
kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change()
retries = kw.get("retries", response.retries)
retries = kw.get("retries")
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect)
@@ -480,11 +399,10 @@ class PoolManager(RequestMethods):
if retries.remove_headers_on_redirect and not conn.is_same_host(
redirect_location
):
new_headers = kw["headers"].copy()
for header in kw["headers"]:
headers = list(six.iterkeys(kw["headers"]))
for header in headers:
if header.lower() in retries.remove_headers_on_redirect:
new_headers.pop(header, None)
kw["headers"] = new_headers
kw["headers"].pop(header, None)
try:
retries = retries.increment(method, url, response=response, _pool=conn)
@@ -530,53 +448,37 @@ class ProxyManager(PoolManager):
private. IP address, target hostname, SNI, and port are always visible
to an HTTPS proxy even when this flag is disabled.
:param proxy_assert_hostname:
The hostname of the certificate to verify against.
:param proxy_assert_fingerprint:
The fingerprint of the certificate to verify against.
Example:
.. code-block:: python
import urllib3
proxy = urllib3.ProxyManager("https://localhost:3128/")
resp1 = proxy.request("GET", "http://google.com/")
resp2 = proxy.request("GET", "http://httpbin.org/")
# One pool was shared by both plain HTTP requests.
print(len(proxy.pools))
# 1
resp3 = proxy.request("GET", "https://httpbin.org/")
resp4 = proxy.request("GET", "https://twitter.com/")
# A separate pool was added for each HTTPS target.
print(len(proxy.pools))
# 3
>>> proxy = urllib3.ProxyManager('http://localhost:3128/')
>>> r1 = proxy.request('GET', 'http://google.com/')
>>> r2 = proxy.request('GET', 'http://httpbin.org/')
>>> len(proxy.pools)
1
>>> r3 = proxy.request('GET', 'https://httpbin.org/')
>>> r4 = proxy.request('GET', 'https://twitter.com/')
>>> len(proxy.pools)
3
"""
def __init__(
self,
proxy_url: str,
num_pools: int = 10,
headers: typing.Mapping[str, str] | None = None,
proxy_headers: typing.Mapping[str, str] | None = None,
proxy_ssl_context: ssl.SSLContext | None = None,
use_forwarding_for_https: bool = False,
proxy_assert_hostname: None | str | typing.Literal[False] = None,
proxy_assert_fingerprint: str | None = None,
**connection_pool_kw: typing.Any,
) -> None:
proxy_url,
num_pools=10,
headers=None,
proxy_headers=None,
proxy_ssl_context=None,
use_forwarding_for_https=False,
**connection_pool_kw
):
if isinstance(proxy_url, HTTPConnectionPool):
str_proxy_url = f"{proxy_url.scheme}://{proxy_url.host}:{proxy_url.port}"
else:
str_proxy_url = proxy_url
proxy = parse_url(str_proxy_url)
proxy_url = "%s://%s:%i" % (
proxy_url.scheme,
proxy_url.host,
proxy_url.port,
)
proxy = parse_url(proxy_url)
if proxy.scheme not in ("http", "https"):
raise ProxySchemeUnknown(proxy.scheme)
@@ -588,38 +490,25 @@ class ProxyManager(PoolManager):
self.proxy = proxy
self.proxy_headers = proxy_headers or {}
self.proxy_ssl_context = proxy_ssl_context
self.proxy_config = ProxyConfig(
proxy_ssl_context,
use_forwarding_for_https,
proxy_assert_hostname,
proxy_assert_fingerprint,
)
self.proxy_config = ProxyConfig(proxy_ssl_context, use_forwarding_for_https)
connection_pool_kw["_proxy"] = self.proxy
connection_pool_kw["_proxy_headers"] = self.proxy_headers
connection_pool_kw["_proxy_config"] = self.proxy_config
super().__init__(num_pools, headers, **connection_pool_kw)
super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
def connection_from_host(
self,
host: str | None,
port: int | None = None,
scheme: str | None = "http",
pool_kwargs: dict[str, typing.Any] | None = None,
) -> HTTPConnectionPool:
def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None):
if scheme == "https":
return super().connection_from_host(
return super(ProxyManager, self).connection_from_host(
host, port, scheme, pool_kwargs=pool_kwargs
)
return super().connection_from_host(
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs # type: ignore[union-attr]
return super(ProxyManager, self).connection_from_host(
self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs
)
def _set_proxy_headers(
self, url: str, headers: typing.Mapping[str, str] | None = None
) -> typing.Mapping[str, str]:
def _set_proxy_headers(self, url, headers=None):
"""
Sets headers needed by proxies: specifically, the Accept and Host
headers. Only sets headers not provided by the user.
@@ -634,9 +523,7 @@ class ProxyManager(PoolManager):
headers_.update(headers)
return headers_
def urlopen( # type: ignore[override]
self, method: str, url: str, redirect: bool = True, **kw: typing.Any
) -> BaseHTTPResponse:
def urlopen(self, method, url, redirect=True, **kw):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
u = parse_url(url)
if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
@@ -646,8 +533,8 @@ class ProxyManager(PoolManager):
headers = kw.get("headers", self.headers)
kw["headers"] = self._set_proxy_headers(url, headers)
return super().urlopen(method, url, redirect=redirect, **kw)
return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
def proxy_from_url(url: str, **kw: typing.Any) -> ProxyManager:
def proxy_from_url(url, **kw):
return ProxyManager(proxy_url=url, **kw)
@@ -1,2 +0,0 @@
# Instruct type checkers to look for inline type annotations in this package.
# See PEP 561.
File diff suppressed because it is too large Load Diff
@@ -1,39 +1,46 @@
# For backwards compatibility, provide imports that used to be here.
from __future__ import annotations
from __future__ import absolute_import
# For backwards compatibility, provide imports that used to be here.
from .connection import is_connection_dropped
from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers
from .response import is_fp_closed
from .retry import Retry
from .ssl_ import (
ALPN_PROTOCOLS,
HAS_SNI,
IS_PYOPENSSL,
IS_SECURETRANSPORT,
PROTOCOL_TLS,
SSLContext,
assert_fingerprint,
create_urllib3_context,
resolve_cert_reqs,
resolve_ssl_version,
ssl_wrap_socket,
)
from .timeout import Timeout
from .url import Url, parse_url
from .timeout import Timeout, current_time
from .url import Url, get_host, parse_url, split_first
from .wait import wait_for_read, wait_for_write
__all__ = (
"HAS_SNI",
"IS_PYOPENSSL",
"IS_SECURETRANSPORT",
"SSLContext",
"PROTOCOL_TLS",
"ALPN_PROTOCOLS",
"Retry",
"Timeout",
"Url",
"assert_fingerprint",
"create_urllib3_context",
"current_time",
"is_connection_dropped",
"is_fp_closed",
"get_host",
"parse_url",
"make_headers",
"resolve_cert_reqs",
"resolve_ssl_version",
"split_first",
"ssl_wrap_socket",
"wait_for_read",
"wait_for_write",
@@ -1,23 +1,33 @@
from __future__ import annotations
from __future__ import absolute_import
import socket
import typing
from ..contrib import _appengine_environ
from ..exceptions import LocationParseError
from .timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT
_TYPE_SOCKET_OPTIONS = list[tuple[int, int, typing.Union[int, bytes]]]
if typing.TYPE_CHECKING:
from .._base_connection import BaseHTTPConnection
from ..packages import six
from .wait import NoWayToWaitForSocketError, wait_for_read
def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specific
def is_connection_dropped(conn): # Platform-specific
"""
Returns True if the connection is dropped and should be closed.
:param conn: :class:`urllib3.connection.HTTPConnection` object.
:param conn:
:class:`http.client.HTTPConnection` object.
Note: For platforms like AppEngine, this will always return ``False`` to
let the platform handle connection recycling transparently for us.
"""
return not conn.is_connected
sock = getattr(conn, "sock", False)
if sock is False: # Platform-specific: AppEngine
return False
if sock is None: # Connection already closed (such as by httplib).
return True
try:
# Returns True if readable, which here means it's been dropped
return wait_for_read(sock, timeout=0.0)
except NoWayToWaitForSocketError: # Platform-specific: AppEngine
return False
# This function is copied from socket.py in the Python 2.7 standard
@@ -25,11 +35,11 @@ def is_connection_dropped(conn: BaseHTTPConnection) -> bool: # Platform-specifi
# One additional modification is that we avoid binding to IPv6 servers
# discovered in DNS if the system doesn't have IPv6 functionality.
def create_connection(
address: tuple[str, int],
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
source_address: tuple[str, int] | None = None,
socket_options: _TYPE_SOCKET_OPTIONS | None = None,
) -> socket.socket:
address,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
source_address=None,
socket_options=None,
):
"""Connect to *address* and return the socket object.
Convenience function. Connect to *address* (a 2-tuple ``(host,
@@ -55,7 +65,9 @@ def create_connection(
try:
host.encode("idna")
except UnicodeError:
raise LocationParseError(f"'{host}', label empty or too long") from None
return six.raise_from(
LocationParseError(u"'%s', label empty or too long" % host), None
)
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
@@ -66,33 +78,26 @@ def create_connection(
# If provided, set socket level options before connecting.
_set_socket_options(sock, socket_options)
if timeout is not _DEFAULT_TIMEOUT:
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
sock.settimeout(timeout)
if source_address:
sock.bind(source_address)
sock.connect(sa)
# Break explicitly a reference cycle
err = None
return sock
except OSError as _:
err = _
except socket.error as e:
err = e
if sock is not None:
sock.close()
sock = None
if err is not None:
try:
raise err
finally:
# Break explicitly a reference cycle
err = None
else:
raise OSError("getaddrinfo returns an empty list")
raise err
raise socket.error("getaddrinfo returns an empty list")
def _set_socket_options(
sock: socket.socket, options: _TYPE_SOCKET_OPTIONS | None
) -> None:
def _set_socket_options(sock, options):
if options is None:
return
@@ -100,7 +105,7 @@ def _set_socket_options(
sock.setsockopt(*opt)
def allowed_gai_family() -> socket.AddressFamily:
def allowed_gai_family():
"""This function is designed to work in the context of
getaddrinfo, where family=socket.AF_UNSPEC is the default and
will perform a DNS search for both IPv6 and IPv4 records."""
@@ -111,11 +116,18 @@ def allowed_gai_family() -> socket.AddressFamily:
return family
def _has_ipv6(host: str) -> bool:
def _has_ipv6(host):
"""Returns True if the system can bind an IPv6 address."""
sock = None
has_ipv6 = False
# App Engine doesn't support IPV6 sockets and actually has a quota on the
# number of sockets that can be used, so just early out here instead of
# creating a socket needlessly.
# See https://github.com/urllib3/urllib3/issues/1446
if _appengine_environ.is_appengine_sandbox():
return False
if socket.has_ipv6:
# has_ipv6 returns true if cPython was compiled with IPv6 support.
# It does not tell us if the system has IPv6 support enabled. To
@@ -1,18 +1,9 @@
from __future__ import annotations
import typing
from .url import Url
if typing.TYPE_CHECKING:
from ..connection import ProxyConfig
from .ssl_ import create_urllib3_context, resolve_cert_reqs, resolve_ssl_version
def connection_requires_http_tunnel(
proxy_url: Url | None = None,
proxy_config: ProxyConfig | None = None,
destination_scheme: str | None = None,
) -> bool:
proxy_url=None, proxy_config=None, destination_scheme=None
):
"""
Returns True if the connection requires an HTTP CONNECT through the proxy.
@@ -41,3 +32,26 @@ def connection_requires_http_tunnel(
# Otherwise always use a tunnel.
return True
def create_proxy_ssl_context(
ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None
):
"""
Generates a default proxy ssl context if one hasn't been provided by the
user.
"""
ssl_context = create_urllib3_context(
ssl_version=resolve_ssl_version(ssl_version),
cert_reqs=resolve_cert_reqs(cert_reqs),
)
if (
not ca_certs
and not ca_cert_dir
and not ca_cert_data
and hasattr(ssl_context, "load_default_certs")
):
ssl_context.load_default_certs()
return ssl_context
@@ -1,16 +1,9 @@
from __future__ import annotations
from __future__ import absolute_import
import io
import sys
import typing
from base64 import b64encode
from enum import Enum
from ..exceptions import UnrewindableBodyError
from .util import to_bytes
if typing.TYPE_CHECKING:
from typing import Final
from ..packages.six import b, integer_types
# Pass as a value within ``headers`` to skip
# emitting some HTTP headers that are added automatically.
@@ -22,49 +15,25 @@ SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"])
ACCEPT_ENCODING = "gzip,deflate"
try:
try:
import brotlicffi as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401
import brotlicffi as _unused_module_brotli # noqa: F401
except ImportError:
import brotli as _unused_module_brotli # type: ignore[import-not-found] # noqa: F401
import brotli as _unused_module_brotli # noqa: F401
except ImportError:
pass
else:
ACCEPT_ENCODING += ",br"
try:
if sys.version_info >= (3, 14):
from compression import zstd as _unused_module_zstd # noqa: F401
else:
from backports import zstd as _unused_module_zstd # noqa: F401
except ImportError:
pass
else:
ACCEPT_ENCODING += ",zstd"
class _TYPE_FAILEDTELL(Enum):
token = 0
_FAILEDTELL: Final[_TYPE_FAILEDTELL] = _TYPE_FAILEDTELL.token
_TYPE_BODY_POSITION = typing.Union[int, _TYPE_FAILEDTELL]
# When sending a request with these methods we aren't expecting
# a body so don't need to set an explicit 'Content-Length: 0'
# The reason we do this in the negative instead of tracking methods
# which 'should' have a body is because unknown methods should be
# treated as if they were 'POST' which *does* expect a body.
_METHODS_NOT_EXPECTING_BODY = {"GET", "HEAD", "DELETE", "TRACE", "OPTIONS", "CONNECT"}
_FAILEDTELL = object()
def make_headers(
keep_alive: bool | None = None,
accept_encoding: bool | list[str] | str | None = None,
user_agent: str | None = None,
basic_auth: str | None = None,
proxy_basic_auth: str | None = None,
disable_cache: bool | None = None,
) -> dict[str, str]:
keep_alive=None,
accept_encoding=None,
user_agent=None,
basic_auth=None,
proxy_basic_auth=None,
disable_cache=None,
):
"""
Shortcuts for generating request headers.
@@ -73,11 +42,7 @@ def make_headers(
:param accept_encoding:
Can be a boolean, list, or string.
``True`` translates to 'gzip,deflate'. If the dependencies for
Brotli (either the ``brotli`` or ``brotlicffi`` package) and/or
Zstandard (the ``backports.zstd`` package for Python before 3.14)
algorithms are installed, then their encodings are
included in the string ('br' and 'zstd', respectively).
``True`` translates to 'gzip,deflate'.
List will get joined by comma.
String will be used as provided.
@@ -96,18 +61,14 @@ def make_headers(
:param disable_cache:
If ``True``, adds 'cache-control: no-cache' header.
Example:
Example::
.. code-block:: python
import urllib3
print(urllib3.util.make_headers(keep_alive=True, user_agent="Batman/1.0"))
# {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
print(urllib3.util.make_headers(accept_encoding=True))
# {'accept-encoding': 'gzip,deflate'}
>>> make_headers(keep_alive=True, user_agent="Batman/1.0")
{'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
>>> make_headers(accept_encoding=True)
{'accept-encoding': 'gzip,deflate'}
"""
headers: dict[str, str] = {}
headers = {}
if accept_encoding:
if isinstance(accept_encoding, str):
pass
@@ -124,14 +85,12 @@ def make_headers(
headers["connection"] = "keep-alive"
if basic_auth:
headers["authorization"] = (
f"Basic {b64encode(basic_auth.encode('latin-1')).decode()}"
)
headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8")
if proxy_basic_auth:
headers["proxy-authorization"] = (
f"Basic {b64encode(proxy_basic_auth.encode('latin-1')).decode()}"
)
headers["proxy-authorization"] = "Basic " + b64encode(
b(proxy_basic_auth)
).decode("utf-8")
if disable_cache:
headers["cache-control"] = "no-cache"
@@ -139,9 +98,7 @@ def make_headers(
return headers
def set_file_position(
body: typing.Any, pos: _TYPE_BODY_POSITION | None
) -> _TYPE_BODY_POSITION | None:
def set_file_position(body, pos):
"""
If a position is provided, move file to that point.
Otherwise, we'll attempt to record a position for future use.
@@ -151,7 +108,7 @@ def set_file_position(
elif getattr(body, "tell", None) is not None:
try:
pos = body.tell()
except OSError:
except (IOError, OSError):
# This differentiates from None, allowing us to catch
# a failed `tell()` later when trying to rewind the body.
pos = _FAILEDTELL
@@ -159,7 +116,7 @@ def set_file_position(
return pos
def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -> None:
def rewind_body(body, body_pos):
"""
Attempt to rewind body to a certain position.
Primarily used for request redirects and retries.
@@ -171,13 +128,13 @@ def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -
Position to seek to in file.
"""
body_seek = getattr(body, "seek", None)
if body_seek is not None and isinstance(body_pos, int):
if body_seek is not None and isinstance(body_pos, integer_types):
try:
body_seek(body_pos)
except OSError as e:
except (IOError, OSError):
raise UnrewindableBodyError(
"An error occurred when rewinding request body for redirect/retry."
) from e
)
elif body_pos is _FAILEDTELL:
raise UnrewindableBodyError(
"Unable to record file position for rewinding "
@@ -185,79 +142,5 @@ def rewind_body(body: typing.IO[typing.AnyStr], body_pos: _TYPE_BODY_POSITION) -
)
else:
raise ValueError(
f"body_pos must be of type integer, instead it was {type(body_pos)}."
"body_pos must be of type integer, instead it was %s." % type(body_pos)
)
class ChunksAndContentLength(typing.NamedTuple):
chunks: typing.Iterable[bytes] | None
content_length: int | None
def body_to_chunks(
body: typing.Any | None, method: str, blocksize: int
) -> ChunksAndContentLength:
"""Takes the HTTP request method, body, and blocksize and
transforms them into an iterable of chunks to pass to
socket.sendall() and an optional 'Content-Length' header.
A 'Content-Length' of 'None' indicates the length of the body
can't be determined so should use 'Transfer-Encoding: chunked'
for framing instead.
"""
chunks: typing.Iterable[bytes] | None
content_length: int | None
# No body, we need to make a recommendation on 'Content-Length'
# based on whether that request method is expected to have
# a body or not.
if body is None:
chunks = None
if method.upper() not in _METHODS_NOT_EXPECTING_BODY:
content_length = 0
else:
content_length = None
# Bytes or strings become bytes
elif isinstance(body, (str, bytes)):
chunks = (to_bytes(body),)
content_length = len(chunks[0])
# File-like object, TODO: use seek() and tell() for length?
elif hasattr(body, "read"):
def chunk_readable() -> typing.Iterable[bytes]:
encode = isinstance(body, io.TextIOBase)
while True:
datablock = body.read(blocksize)
if not datablock:
break
if encode:
datablock = datablock.encode("utf-8")
yield datablock
chunks = chunk_readable()
content_length = None
# Otherwise we need to start checking via duck-typing.
else:
try:
# Check if the body implements the buffer API.
mv = memoryview(body)
except TypeError:
try:
# Check if the body is an iterable
chunks = iter(body)
content_length = None
except TypeError:
raise TypeError(
f"'body' must be a bytes-like object, file-like "
f"object, or iterable. Instead was {body!r}"
) from None
else:
# Since it implements the buffer API can be passed directly to socket.sendall()
chunks = (body,)
content_length = mv.nbytes
return ChunksAndContentLength(chunks=chunks, content_length=content_length)
@@ -1,12 +1,12 @@
from __future__ import annotations
from __future__ import absolute_import
import http.client as httplib
from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect
from ..exceptions import HeaderParsingError
from ..packages.six.moves import http_client as httplib
def is_fp_closed(obj: object) -> bool:
def is_fp_closed(obj):
"""
Checks whether a given file-like object is closed.
@@ -17,27 +17,27 @@ def is_fp_closed(obj: object) -> bool:
try:
# Check `isclosed()` first, in case Python3 doesn't set `closed`.
# GH Issue #928
return obj.isclosed() # type: ignore[no-any-return, attr-defined]
return obj.isclosed()
except AttributeError:
pass
try:
# Check via the official file-like-object way.
return obj.closed # type: ignore[no-any-return, attr-defined]
return obj.closed
except AttributeError:
pass
try:
# Check if the object is a container for another file-like object that
# gets released on exhaustion (e.g. HTTPResponse).
return obj.fp is None # type: ignore[attr-defined]
return obj.fp is None
except AttributeError:
pass
raise ValueError("Unable to determine whether fp is closed.")
def assert_header_parsing(headers: httplib.HTTPMessage) -> None:
def assert_header_parsing(headers):
"""
Asserts whether all headers have been successfully parsed.
Extracts encountered errors from the result of parsing headers.
@@ -53,49 +53,55 @@ def assert_header_parsing(headers: httplib.HTTPMessage) -> None:
# This will fail silently if we pass in the wrong kind of parameter.
# To make debugging easier add an explicit check.
if not isinstance(headers, httplib.HTTPMessage):
raise TypeError(f"expected httplib.Message, got {type(headers)}.")
raise TypeError("expected httplib.Message, got {0}.".format(type(headers)))
defects = getattr(headers, "defects", None)
get_payload = getattr(headers, "get_payload", None)
unparsed_data = None
if get_payload:
# get_payload is actually email.message.Message.get_payload;
# we're only interested in the result if it's not a multipart message
if not headers.is_multipart():
payload = get_payload()
# get_payload is actually email.message.Message.get_payload;
# we're only interested in the result if it's not a multipart message
if not headers.is_multipart():
payload = headers.get_payload()
if isinstance(payload, (bytes, str)):
unparsed_data = payload
if defects:
# httplib is assuming a response body is available
# when parsing headers even when httplib only sends
# header data to parse_headers() This results in
# defects on multipart responses in particular.
# See: https://github.com/urllib3/urllib3/issues/800
if isinstance(payload, (bytes, str)):
unparsed_data = payload
# httplib is assuming a response body is available
# when parsing headers even when httplib only sends
# header data to parse_headers() This results in
# defects on multipart responses in particular.
# See: https://github.com/urllib3/urllib3/issues/800
# So we ignore the following defects:
# - StartBoundaryNotFoundDefect:
# The claimed start boundary was never found.
# - MultipartInvariantViolationDefect:
# A message claimed to be a multipart but no subparts were found.
defects = [
defect
for defect in headers.defects
if not isinstance(
defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect)
)
]
# So we ignore the following defects:
# - StartBoundaryNotFoundDefect:
# The claimed start boundary was never found.
# - MultipartInvariantViolationDefect:
# A message claimed to be a multipart but no subparts were found.
defects = [
defect
for defect in defects
if not isinstance(
defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect)
)
]
if defects or unparsed_data:
raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)
def is_response_to_head(response: httplib.HTTPResponse) -> bool:
def is_response_to_head(response):
"""
Checks whether the request of a response has been a HEAD-request.
Handles the quirks of AppEngine.
:param http.client.HTTPResponse response:
Response to check if the originating request
used 'HEAD' as a method.
"""
# FIXME: Can we do this somehow without accessing private httplib _method?
method_str = response._method # type: str # type: ignore[attr-defined]
return method_str.upper() == "HEAD"
method = response._method
if isinstance(method, int): # Platform-specific: Appengine
return method == 3
return method.upper() == "HEAD"
@@ -1,13 +1,12 @@
from __future__ import annotations
from __future__ import absolute_import
import email
import logging
import random
import re
import time
import typing
import warnings
from collections import namedtuple
from itertools import takewhile
from types import TracebackType
from ..exceptions import (
ConnectTimeoutError,
@@ -18,51 +17,97 @@ from ..exceptions import (
ReadTimeoutError,
ResponseError,
)
from .util import reraise
if typing.TYPE_CHECKING:
from typing_extensions import Self
from ..connectionpool import ConnectionPool
from ..response import BaseHTTPResponse
from ..packages import six
log = logging.getLogger(__name__)
# Data structure for representing the metadata of requests that result in a retry.
class RequestHistory(typing.NamedTuple):
method: str | None
url: str | None
error: Exception | None
status: int | None
redirect_location: str | None
RequestHistory = namedtuple(
"RequestHistory", ["method", "url", "error", "status", "redirect_location"]
)
class Retry:
# TODO: In v2 we can remove this sentinel and metaclass with deprecated options.
_Default = object()
class _RetryMeta(type):
@property
def DEFAULT_METHOD_WHITELIST(cls):
warnings.warn(
"Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
"will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
DeprecationWarning,
)
return cls.DEFAULT_ALLOWED_METHODS
@DEFAULT_METHOD_WHITELIST.setter
def DEFAULT_METHOD_WHITELIST(cls, value):
warnings.warn(
"Using 'Retry.DEFAULT_METHOD_WHITELIST' is deprecated and "
"will be removed in v2.0. Use 'Retry.DEFAULT_ALLOWED_METHODS' instead",
DeprecationWarning,
)
cls.DEFAULT_ALLOWED_METHODS = value
@property
def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls):
warnings.warn(
"Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
"will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
DeprecationWarning,
)
return cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
@DEFAULT_REDIRECT_HEADERS_BLACKLIST.setter
def DEFAULT_REDIRECT_HEADERS_BLACKLIST(cls, value):
warnings.warn(
"Using 'Retry.DEFAULT_REDIRECT_HEADERS_BLACKLIST' is deprecated and "
"will be removed in v2.0. Use 'Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT' instead",
DeprecationWarning,
)
cls.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = value
@property
def BACKOFF_MAX(cls):
warnings.warn(
"Using 'Retry.BACKOFF_MAX' is deprecated and "
"will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
DeprecationWarning,
)
return cls.DEFAULT_BACKOFF_MAX
@BACKOFF_MAX.setter
def BACKOFF_MAX(cls, value):
warnings.warn(
"Using 'Retry.BACKOFF_MAX' is deprecated and "
"will be removed in v2.0. Use 'Retry.DEFAULT_BACKOFF_MAX' instead",
DeprecationWarning,
)
cls.DEFAULT_BACKOFF_MAX = value
@six.add_metaclass(_RetryMeta)
class Retry(object):
"""Retry configuration.
Each retry attempt will create a new Retry object with updated values, so
they can be safely reused.
Retries can be defined as a default for a pool:
.. code-block:: python
Retries can be defined as a default for a pool::
retries = Retry(connect=5, read=2, redirect=5)
http = PoolManager(retries=retries)
response = http.request("GET", "https://example.com/")
response = http.request('GET', 'http://example.com/')
Or per-request (which overrides the default for the pool):
Or per-request (which overrides the default for the pool)::
.. code-block:: python
response = http.request('GET', 'http://example.com/', retries=Retry(10))
response = http.request("GET", "https://example.com/", retries=Retry(10))
Retries can be disabled by passing ``False``::
Retries can be disabled by passing ``False``:
.. code-block:: python
response = http.request("GET", "https://example.com/", retries=False)
response = http.request('GET', 'http://example.com/', retries=False)
Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
retries are disabled, in which case the causing exception will be raised.
@@ -124,16 +169,21 @@ class Retry:
If ``total`` is not set, it's a good idea to set this to 0 to account
for unexpected edge cases and avoid infinite retry loops.
:param Collection allowed_methods:
:param iterable allowed_methods:
Set of uppercased HTTP method verbs that we should retry on.
By default, we only retry on methods which are considered to be
idempotent (multiple requests with the same parameters end with the
same state). See :attr:`Retry.DEFAULT_ALLOWED_METHODS`.
Set to a ``None`` value to retry on any verb.
Set to a ``False`` value to retry on any verb.
:param Collection status_forcelist:
.. warning::
Previously this parameter was named ``method_whitelist``, that
usage is deprecated in v1.26.0 and will be removed in v2.0.
:param iterable status_forcelist:
A set of integer HTTP status codes that we should force a retry on.
A retry is initiated if the request method is in ``allowed_methods``
and the response status code is in ``status_forcelist``.
@@ -145,25 +195,13 @@ class Retry:
(most errors are resolved immediately by a second try without a
delay). urllib3 will sleep for::
{backoff factor} * (2 ** ({number of previous retries}))
{backoff factor} * (2 ** ({number of total retries} - 1))
seconds. If `backoff_jitter` is non-zero, this sleep is extended by::
seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
than :attr:`Retry.DEFAULT_BACKOFF_MAX`.
random.uniform(0, {backoff jitter})
seconds. For example, if the backoff_factor is 0.1, then :func:`Retry.sleep` will
sleep for [0.0s, 0.2s, 0.4s, 0.8s, ...] between retries. No backoff will ever
be longer than `backoff_max`.
By default, backoff is disabled (factor set to 0).
:param float backoff_max:
The maximum backoff time (in seconds) between retry attempts.
This value caps the computed backoff from `backoff_factor`.
:param float backoff_jitter:
Random jitter amount (in seconds) added to the computed backoff.
Jitter is sampled uniformly from `0` to `backoff_jitter`.
By default, backoff is disabled (set to 0).
:param bool raise_on_redirect: Whether, if the number of redirects is
exhausted, to raise a MaxRetryError, or to return a response with a
@@ -182,15 +220,10 @@ class Retry:
Whether to respect Retry-After header on status codes defined as
:attr:`Retry.RETRY_AFTER_STATUS_CODES` or not.
:param Collection remove_headers_on_redirect:
:param iterable remove_headers_on_redirect:
Sequence of headers to remove from the request when a response
indicating a redirect is returned before firing off the redirected
request.
:param int retry_after_max: Number of seconds to allow as the maximum for
Retry-After headers. Defaults to :attr:`Retry.DEFAULT_RETRY_AFTER_MAX`.
Any Retry-After headers larger than this value will be limited to this
value.
"""
#: Default methods to be used for ``allowed_methods``
@@ -206,38 +239,48 @@ class Retry:
["Cookie", "Authorization", "Proxy-Authorization"]
)
#: Default maximum backoff time.
#: Maximum backoff time.
DEFAULT_BACKOFF_MAX = 120
# This is undocumented in the RFC. Setting to 6 hours matches other popular libraries.
#: Default maximum allowed value for Retry-After headers in seconds
DEFAULT_RETRY_AFTER_MAX: typing.Final[int] = 21600
# Backward compatibility; assigned outside of the class.
DEFAULT: typing.ClassVar[Retry]
def __init__(
self,
total: bool | int | None = 10,
connect: int | None = None,
read: int | None = None,
redirect: bool | int | None = None,
status: int | None = None,
other: int | None = None,
allowed_methods: typing.Collection[str] | None = DEFAULT_ALLOWED_METHODS,
status_forcelist: typing.Collection[int] | None = None,
backoff_factor: float = 0,
backoff_max: float = DEFAULT_BACKOFF_MAX,
raise_on_redirect: bool = True,
raise_on_status: bool = True,
history: tuple[RequestHistory, ...] | None = None,
respect_retry_after_header: bool = True,
remove_headers_on_redirect: typing.Collection[
str
] = DEFAULT_REMOVE_HEADERS_ON_REDIRECT,
backoff_jitter: float = 0.0,
retry_after_max: int = DEFAULT_RETRY_AFTER_MAX,
) -> None:
total=10,
connect=None,
read=None,
redirect=None,
status=None,
other=None,
allowed_methods=_Default,
status_forcelist=None,
backoff_factor=0,
raise_on_redirect=True,
raise_on_status=True,
history=None,
respect_retry_after_header=True,
remove_headers_on_redirect=_Default,
# TODO: Deprecated, remove in v2.0
method_whitelist=_Default,
):
if method_whitelist is not _Default:
if allowed_methods is not _Default:
raise ValueError(
"Using both 'allowed_methods' and "
"'method_whitelist' together is not allowed. "
"Instead only use 'allowed_methods'"
)
warnings.warn(
"Using 'method_whitelist' with Retry is deprecated and "
"will be removed in v2.0. Use 'allowed_methods' instead",
DeprecationWarning,
stacklevel=2,
)
allowed_methods = method_whitelist
if allowed_methods is _Default:
allowed_methods = self.DEFAULT_ALLOWED_METHODS
if remove_headers_on_redirect is _Default:
remove_headers_on_redirect = self.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
self.total = total
self.connect = connect
self.read = read
@@ -252,18 +295,15 @@ class Retry:
self.status_forcelist = status_forcelist or set()
self.allowed_methods = allowed_methods
self.backoff_factor = backoff_factor
self.backoff_max = backoff_max
self.retry_after_max = retry_after_max
self.raise_on_redirect = raise_on_redirect
self.raise_on_status = raise_on_status
self.history = history or ()
self.history = history or tuple()
self.respect_retry_after_header = respect_retry_after_header
self.remove_headers_on_redirect = frozenset(
h.lower() for h in remove_headers_on_redirect
[h.lower() for h in remove_headers_on_redirect]
)
self.backoff_jitter = backoff_jitter
def new(self, **kw: typing.Any) -> Self:
def new(self, **kw):
params = dict(
total=self.total,
connect=self.connect,
@@ -271,29 +311,36 @@ class Retry:
redirect=self.redirect,
status=self.status,
other=self.other,
allowed_methods=self.allowed_methods,
status_forcelist=self.status_forcelist,
backoff_factor=self.backoff_factor,
backoff_max=self.backoff_max,
retry_after_max=self.retry_after_max,
raise_on_redirect=self.raise_on_redirect,
raise_on_status=self.raise_on_status,
history=self.history,
remove_headers_on_redirect=self.remove_headers_on_redirect,
respect_retry_after_header=self.respect_retry_after_header,
backoff_jitter=self.backoff_jitter,
)
# TODO: If already given in **kw we use what's given to us
# If not given we need to figure out what to pass. We decide
# based on whether our class has the 'method_whitelist' property
# and if so we pass the deprecated 'method_whitelist' otherwise
# we use 'allowed_methods'. Remove in v2.0
if "method_whitelist" not in kw and "allowed_methods" not in kw:
if "method_whitelist" in self.__dict__:
warnings.warn(
"Using 'method_whitelist' with Retry is deprecated and "
"will be removed in v2.0. Use 'allowed_methods' instead",
DeprecationWarning,
)
params["method_whitelist"] = self.allowed_methods
else:
params["allowed_methods"] = self.allowed_methods
params.update(kw)
return type(self)(**params) # type: ignore[arg-type]
return type(self)(**params)
@classmethod
def from_int(
cls,
retries: Retry | bool | int | None,
redirect: bool | int | None = True,
default: Retry | bool | int | None = None,
) -> Retry:
def from_int(cls, retries, redirect=True, default=None):
"""Backwards-compatibility for the old retries format."""
if retries is None:
retries = default if default is not None else cls.DEFAULT
@@ -306,7 +353,7 @@ class Retry:
log.debug("Converted retries value: %r -> %r", retries, new_retries)
return new_retries
def get_backoff_time(self) -> float:
def get_backoff_time(self):
"""Formula for computing the current backoff
:rtype: float
@@ -321,32 +368,32 @@ class Retry:
return 0
backoff_value = self.backoff_factor * (2 ** (consecutive_errors_len - 1))
if self.backoff_jitter != 0.0:
backoff_value += random.random() * self.backoff_jitter
return float(max(0, min(self.backoff_max, backoff_value)))
return min(self.DEFAULT_BACKOFF_MAX, backoff_value)
def parse_retry_after(self, retry_after: str) -> float:
seconds: float
def parse_retry_after(self, retry_after):
# Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4
if re.match(r"^\s*[0-9]+\s*$", retry_after):
seconds = int(retry_after)
else:
retry_date_tuple = email.utils.parsedate_tz(retry_after)
if retry_date_tuple is None:
raise InvalidHeader(f"Invalid Retry-After header: {retry_after}")
raise InvalidHeader("Invalid Retry-After header: %s" % retry_after)
if retry_date_tuple[9] is None: # Python 2
# Assume UTC if no timezone was specified
# On Python2.7, parsedate_tz returns None for a timezone offset
# instead of 0 if no timezone is given, where mktime_tz treats
# a None timezone offset as local time.
retry_date_tuple = retry_date_tuple[:9] + (0,) + retry_date_tuple[10:]
retry_date = email.utils.mktime_tz(retry_date_tuple)
seconds = retry_date - time.time()
seconds = max(seconds, 0)
# Check the seconds do not exceed the specified maximum
if seconds > self.retry_after_max:
seconds = self.retry_after_max
if seconds < 0:
seconds = 0
return seconds
def get_retry_after(self, response: BaseHTTPResponse) -> float | None:
def get_retry_after(self, response):
"""Get the value of Retry-After in seconds."""
retry_after = response.headers.get("Retry-After")
@@ -356,7 +403,7 @@ class Retry:
return self.parse_retry_after(retry_after)
def sleep_for_retry(self, response: BaseHTTPResponse) -> bool:
def sleep_for_retry(self, response=None):
retry_after = self.get_retry_after(response)
if retry_after:
time.sleep(retry_after)
@@ -364,13 +411,13 @@ class Retry:
return False
def _sleep_backoff(self) -> None:
def _sleep_backoff(self):
backoff = self.get_backoff_time()
if backoff <= 0:
return
time.sleep(backoff)
def sleep(self, response: BaseHTTPResponse | None = None) -> None:
def sleep(self, response=None):
"""Sleep between retry attempts.
This method will respect a server's ``Retry-After`` response header
@@ -386,7 +433,7 @@ class Retry:
self._sleep_backoff()
def _is_connection_error(self, err: Exception) -> bool:
def _is_connection_error(self, err):
"""Errors when we're fairly sure that the server did not receive the
request, so it should be safe to retry.
"""
@@ -394,23 +441,33 @@ class Retry:
err = err.original_error
return isinstance(err, ConnectTimeoutError)
def _is_read_error(self, err: Exception) -> bool:
def _is_read_error(self, err):
"""Errors that occur after the request has been started, so we should
assume that the server began processing it.
"""
return isinstance(err, (ReadTimeoutError, ProtocolError))
def _is_method_retryable(self, method: str) -> bool:
def _is_method_retryable(self, method):
"""Checks if a given HTTP method should be retried upon, depending if
it is included in the allowed_methods
"""
if self.allowed_methods and method.upper() not in self.allowed_methods:
# TODO: For now favor if the Retry implementation sets its own method_whitelist
# property outside of our constructor to avoid breaking custom implementations.
if "method_whitelist" in self.__dict__:
warnings.warn(
"Using 'method_whitelist' with Retry is deprecated and "
"will be removed in v2.0. Use 'allowed_methods' instead",
DeprecationWarning,
)
allowed_methods = self.method_whitelist
else:
allowed_methods = self.allowed_methods
if allowed_methods and method.upper() not in allowed_methods:
return False
return True
def is_retry(
self, method: str, status_code: int, has_retry_after: bool = False
) -> bool:
def is_retry(self, method, status_code, has_retry_after=False):
"""Is this method/status code retryable? (Based on allowlists and control
variables such as the number of total retries to allow, whether to
respect the Retry-After header, whether this header is present, and
@@ -423,27 +480,24 @@ class Retry:
if self.status_forcelist and status_code in self.status_forcelist:
return True
return bool(
return (
self.total
and self.respect_retry_after_header
and has_retry_after
and (status_code in self.RETRY_AFTER_STATUS_CODES)
)
def is_exhausted(self) -> bool:
def is_exhausted(self):
"""Are we out of retries?"""
retry_counts = [
x
for x in (
self.total,
self.connect,
self.read,
self.redirect,
self.status,
self.other,
)
if x
]
retry_counts = (
self.total,
self.connect,
self.read,
self.redirect,
self.status,
self.other,
)
retry_counts = list(filter(None, retry_counts))
if not retry_counts:
return False
@@ -451,18 +505,18 @@ class Retry:
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
method=None,
url=None,
response=None,
error=None,
_pool=None,
_stacktrace=None,
):
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:type response: :class:`~urllib3.response.HTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
@@ -470,7 +524,7 @@ class Retry:
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
raise six.reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
@@ -488,14 +542,14 @@ class Retry:
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
raise six.reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
if read is False or not self._is_method_retryable(method):
raise six.reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
@@ -509,9 +563,7 @@ class Retry:
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
redirect_location = response.get_redirect_location()
status = response.status
else:
@@ -539,18 +591,31 @@ class Retry:
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
raise MaxRetryError(_pool, url, error or ResponseError(cause))
log.debug("Incremented Retry for (url='%s'): %r", url, new_retry)
return new_retry
def __repr__(self) -> str:
def __repr__(self):
return (
f"{type(self).__name__}(total={self.total}, connect={self.connect}, "
f"read={self.read}, redirect={self.redirect}, status={self.status})"
)
"{cls.__name__}(total={self.total}, connect={self.connect}, "
"read={self.read}, redirect={self.redirect}, status={self.status})"
).format(cls=type(self), self=self)
def __getattr__(self, item):
if item == "method_whitelist":
# TODO: Remove this deprecated alias in v2.0
warnings.warn(
"Using 'method_whitelist' with Retry is deprecated and "
"will be removed in v2.0. Use 'allowed_methods' instead",
DeprecationWarning,
)
return self.allowed_methods
try:
return getattr(super(Retry, self), item)
except AttributeError:
return getattr(Retry, item)
# For backwards compatibility (equivalent to pre-v1.9):
@@ -1,25 +1,28 @@
from __future__ import annotations
from __future__ import absolute_import
import hashlib
import hmac
import os
import socket
import sys
import typing
import warnings
from binascii import unhexlify
from binascii import hexlify, unhexlify
from ..exceptions import ProxySchemeUnsupported, SSLError
from .url import _BRACELESS_IPV6_ADDRZ_RE, _IPV4_RE
from ..exceptions import (
InsecurePlatformWarning,
ProxySchemeUnsupported,
SNIMissingWarning,
SSLError,
)
from ..packages import six
from .url import BRACELESS_IPV6_ADDRZ_RE, IPV4_RE
SSLContext = None
SSLTransport = None
HAS_NEVER_CHECK_COMMON_NAME = False
HAS_SNI = False
IS_PYOPENSSL = False
IS_SECURETRANSPORT = False
ALPN_PROTOCOLS = ["http/1.1"]
_TYPE_VERSION_INFO = tuple[int, int, int, str, int]
# Maps the length of a digest to a possible hash function producing this digest
HASHFUNC_MAP = {
length: getattr(hashlib, algorithm, None)
@@ -27,84 +30,159 @@ HASHFUNC_MAP = {
}
def _is_has_never_check_common_name_reliable(
openssl_version: str,
) -> bool:
# As of May 2023, all released versions of LibreSSL fail to reject certificates with
# only common names, see https://github.com/urllib3/urllib3/pull/3024
is_openssl = openssl_version.startswith("OpenSSL ")
def _const_compare_digest_backport(a, b):
"""
Compare two digests of equal length in constant time.
return is_openssl
The digests must be of type str/bytes.
Returns True if the digests match, and False otherwise.
"""
result = abs(len(a) - len(b))
for left, right in zip(bytearray(a), bytearray(b)):
result |= left ^ right
return result == 0
if typing.TYPE_CHECKING:
from ssl import VerifyMode
from typing import TypedDict
_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport)
from .ssltransport import SSLTransport as SSLTransportType
class _TYPE_PEER_CERT_RET_DICT(TypedDict, total=False):
subjectAltName: tuple[tuple[str, str], ...]
subject: tuple[tuple[tuple[str, str], ...], ...]
serialNumber: str
# Mapping from 'ssl.PROTOCOL_TLSX' to 'TLSVersion.X'
_SSL_VERSION_TO_TLS_VERSION: dict[int, int] = {}
try: # Do we have ssl at all?
try: # Test for SSL features
import ssl
from ssl import ( # type: ignore[assignment]
CERT_REQUIRED,
HAS_NEVER_CHECK_COMMON_NAME,
OP_NO_COMPRESSION,
OP_NO_TICKET,
OPENSSL_VERSION,
PROTOCOL_TLS,
PROTOCOL_TLS_CLIENT,
VERIFY_X509_PARTIAL_CHAIN,
VERIFY_X509_STRICT,
OP_NO_SSLv2,
OP_NO_SSLv3,
SSLContext,
TLSVersion,
)
from ssl import CERT_REQUIRED, wrap_socket
except ImportError:
pass
try:
from ssl import HAS_SNI # Has SNI?
except ImportError:
pass
try:
from .ssltransport import SSLTransport
except ImportError:
pass
try: # Platform-specific: Python 3.6
from ssl import PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
# Setting SSLContext.hostname_checks_common_name = False didn't work with
# LibreSSL, check details in the used function.
if HAS_NEVER_CHECK_COMMON_NAME and not _is_has_never_check_common_name_reliable(
OPENSSL_VERSION,
): # Defensive:
HAS_NEVER_CHECK_COMMON_NAME = False
# Need to be careful here in case old TLS versions get
# removed in future 'ssl' module implementations.
for attr in ("TLSv1", "TLSv1_1", "TLSv1_2"):
try:
_SSL_VERSION_TO_TLS_VERSION[getattr(ssl, f"PROTOCOL_{attr}")] = getattr(
TLSVersion, attr
)
except AttributeError: # Defensive:
continue
from .ssltransport import SSLTransport # type: ignore[assignment]
except ImportError:
OP_NO_COMPRESSION = 0x20000 # type: ignore[assignment, misc]
OP_NO_TICKET = 0x4000 # type: ignore[assignment, misc]
OP_NO_SSLv2 = 0x1000000 # type: ignore[assignment, misc]
OP_NO_SSLv3 = 0x2000000 # type: ignore[assignment, misc]
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 # type: ignore[assignment, misc]
PROTOCOL_TLS_CLIENT = 16 # type: ignore[assignment, misc]
VERIFY_X509_PARTIAL_CHAIN = 0x80000 # type: ignore[assignment,misc]
VERIFY_X509_STRICT = 0x20 # type: ignore[assignment, misc]
try:
from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS
PROTOCOL_SSLv23 = PROTOCOL_TLS
except ImportError:
PROTOCOL_SSLv23 = PROTOCOL_TLS = 2
try:
from ssl import PROTOCOL_TLS_CLIENT
except ImportError:
PROTOCOL_TLS_CLIENT = PROTOCOL_TLS
_TYPE_PEER_CERT_RET = typing.Union["_TYPE_PEER_CERT_RET_DICT", bytes, None]
try:
from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3
except ImportError:
OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
OP_NO_COMPRESSION = 0x20000
def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
try: # OP_NO_TICKET was added in Python 3.6
from ssl import OP_NO_TICKET
except ImportError:
OP_NO_TICKET = 0x4000
# A secure default.
# Sources for more information on TLS ciphers:
#
# - https://wiki.mozilla.org/Security/Server_Side_TLS
# - https://www.ssllabs.com/projects/best-practices/index.html
# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
#
# The general intent is:
# - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE),
# - prefer ECDHE over DHE for better performance,
# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and
# security,
# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common,
# - disable NULL authentication, MD5 MACs, DSS, and other
# insecure ciphers for security reasons.
# - NOTE: TLS 1.3 cipher suites are managed through a different interface
# not exposed by CPython (yet!) and are enabled by default if they're available.
DEFAULT_CIPHERS = ":".join(
[
"ECDHE+AESGCM",
"ECDHE+CHACHA20",
"DHE+AESGCM",
"DHE+CHACHA20",
"ECDH+AESGCM",
"DH+AESGCM",
"ECDH+AES",
"DH+AES",
"RSA+AESGCM",
"RSA+AES",
"!aNULL",
"!eNULL",
"!MD5",
"!DSS",
]
)
try:
from ssl import SSLContext # Modern SSL?
except ImportError:
class SSLContext(object): # Platform-specific: Python 2
def __init__(self, protocol_version):
self.protocol = protocol_version
# Use default values from a real SSLContext
self.check_hostname = False
self.verify_mode = ssl.CERT_NONE
self.ca_certs = None
self.options = 0
self.certfile = None
self.keyfile = None
self.ciphers = None
def load_cert_chain(self, certfile, keyfile):
self.certfile = certfile
self.keyfile = keyfile
def load_verify_locations(self, cafile=None, capath=None, cadata=None):
self.ca_certs = cafile
if capath is not None:
raise SSLError("CA directories not supported in older Pythons")
if cadata is not None:
raise SSLError("CA data not supported in older Pythons")
def set_ciphers(self, cipher_suite):
self.ciphers = cipher_suite
def wrap_socket(self, socket, server_hostname=None, server_side=False):
warnings.warn(
"A true SSLContext object is not available. This prevents "
"urllib3 from configuring SSL appropriately and may cause "
"certain SSL connections to fail. You can upgrade to a newer "
"version of Python to solve this. For more information, see "
"https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"
"#ssl-warnings",
InsecurePlatformWarning,
)
kwargs = {
"keyfile": self.keyfile,
"certfile": self.certfile,
"ca_certs": self.ca_certs,
"cert_reqs": self.verify_mode,
"ssl_version": self.protocol,
"server_side": server_side,
}
return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
def assert_fingerprint(cert, fingerprint):
"""
Checks if given fingerprint matches the supplied certificate.
@@ -114,17 +192,16 @@ def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
Fingerprint as string of hexdigits, can be interspersed by colons.
"""
if cert is None:
raise SSLError("No certificate for the peer.")
fingerprint = fingerprint.replace(":", "").lower()
digest_length = len(fingerprint)
if digest_length not in HASHFUNC_MAP:
raise SSLError(f"Fingerprint of invalid length: {fingerprint}")
raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint))
hashfunc = HASHFUNC_MAP.get(digest_length)
if hashfunc is None:
raise SSLError(
f"Hash function implementation unavailable for fingerprint length: {digest_length}"
"Hash function implementation unavailable for fingerprint length: {0}".format(
digest_length
)
)
# We need encode() here for py32; works on py2 and p33.
@@ -132,13 +209,15 @@ def assert_fingerprint(cert: bytes | None, fingerprint: str) -> None:
cert_digest = hashfunc(cert).digest()
if not hmac.compare_digest(cert_digest, fingerprint_bytes):
if not _const_compare_digest(cert_digest, fingerprint_bytes):
raise SSLError(
f'Fingerprints did not match. Expected "{fingerprint}", got "{cert_digest.hex()}"'
'Fingerprints did not match. Expected "{0}", got "{1}".'.format(
fingerprint, hexlify(cert_digest)
)
)
def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode:
def resolve_cert_reqs(candidate):
"""
Resolves the argument to a numeric constant, which can be passed to
the wrap_socket function/method from the ssl module.
@@ -156,12 +235,12 @@ def resolve_cert_reqs(candidate: None | int | str) -> VerifyMode:
res = getattr(ssl, candidate, None)
if res is None:
res = getattr(ssl, "CERT_" + candidate)
return res # type: ignore[no-any-return]
return res
return candidate # type: ignore[return-value]
return candidate
def resolve_ssl_version(candidate: None | int | str) -> int:
def resolve_ssl_version(candidate):
"""
like resolve_cert_reqs
"""
@@ -172,34 +251,35 @@ def resolve_ssl_version(candidate: None | int | str) -> int:
res = getattr(ssl, candidate, None)
if res is None:
res = getattr(ssl, "PROTOCOL_" + candidate)
return typing.cast(int, res)
return res
return candidate
def create_urllib3_context(
ssl_version: int | None = None,
cert_reqs: int | None = None,
options: int | None = None,
ciphers: str | None = None,
ssl_minimum_version: int | None = None,
ssl_maximum_version: int | None = None,
verify_flags: int | None = None,
) -> ssl.SSLContext:
"""Creates and configures an :class:`ssl.SSLContext` instance for use with urllib3.
ssl_version=None, cert_reqs=None, options=None, ciphers=None
):
"""All arguments have the same meaning as ``ssl_wrap_socket``.
By default, this function does a lot of the same work that
``ssl.create_default_context`` does on Python 3.4+. It:
- Disables SSLv2, SSLv3, and compression
- Sets a restricted set of server ciphers
If you wish to enable SSLv3, you can do::
from urllib3.util import ssl_
context = ssl_.create_urllib3_context()
context.options &= ~ssl_.OP_NO_SSLv3
You can do the same to enable compression (substituting ``COMPRESSION``
for ``SSLv3`` in the last line above).
:param ssl_version:
The desired protocol version to use. This will default to
PROTOCOL_SSLv23 which will negotiate the highest protocol that both
the server and your installation of OpenSSL support.
This parameter is deprecated instead use 'ssl_minimum_version'.
:param ssl_minimum_version:
The minimum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
:param ssl_maximum_version:
The maximum version of TLS to be used. Use the 'ssl.TLSVersion' enum for specifying the value.
Not recommended to set to anything other than 'ssl.TLSVersion.MAXIMUM_SUPPORTED' which is the
default value.
:param cert_reqs:
Whether to require the certificate verification. This defaults to
``ssl.CERT_REQUIRED``.
@@ -207,61 +287,18 @@ def create_urllib3_context(
Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``.
:param ciphers:
Which cipher suites to allow the server to select. Defaults to either system configured
ciphers if OpenSSL 1.1.1+, otherwise uses a secure default set of ciphers.
:param verify_flags:
The flags for certificate verification operations. These default to
``ssl.VERIFY_X509_PARTIAL_CHAIN`` and ``ssl.VERIFY_X509_STRICT`` for Python 3.13+.
Which cipher suites to allow the server to select.
:returns:
Constructed SSLContext object with specified options
:rtype: SSLContext
"""
if SSLContext is None:
raise TypeError("Can't create an SSLContext object without an ssl module")
# PROTOCOL_TLS is deprecated in Python 3.10
if not ssl_version or ssl_version == PROTOCOL_TLS:
ssl_version = PROTOCOL_TLS_CLIENT
# This means 'ssl_version' was specified as an exact value.
if ssl_version not in (None, PROTOCOL_TLS, PROTOCOL_TLS_CLIENT):
# Disallow setting 'ssl_version' and 'ssl_minimum|maximum_version'
# to avoid conflicts.
if ssl_minimum_version is not None or ssl_maximum_version is not None:
raise ValueError(
"Can't specify both 'ssl_version' and either "
"'ssl_minimum_version' or 'ssl_maximum_version'"
)
context = SSLContext(ssl_version)
# 'ssl_version' is deprecated and will be removed in the future.
else:
# Use 'ssl_minimum_version' and 'ssl_maximum_version' instead.
ssl_minimum_version = _SSL_VERSION_TO_TLS_VERSION.get(
ssl_version, TLSVersion.MINIMUM_SUPPORTED
)
ssl_maximum_version = _SSL_VERSION_TO_TLS_VERSION.get(
ssl_version, TLSVersion.MAXIMUM_SUPPORTED
)
# This warning message is pushing users to use 'ssl_minimum_version'
# instead of both min/max. Best practice is to only set the minimum version and
# keep the maximum version to be it's default value: 'TLSVersion.MAXIMUM_SUPPORTED'
warnings.warn(
"'ssl_version' option is deprecated and will be "
"removed in urllib3 v3.0. Instead use 'ssl_minimum_version'",
category=FutureWarning,
stacklevel=2,
)
context = SSLContext(PROTOCOL_TLS_CLIENT)
if ssl_minimum_version is not None:
context.minimum_version = ssl_minimum_version
else: # pyOpenSSL defaults to 'MINIMUM_SUPPORTED' so explicitly set TLSv1.2 here
context.minimum_version = TLSVersion.TLSv1_2
if ssl_maximum_version is not None:
context.maximum_version = ssl_maximum_version
# Unless we're given ciphers defer to either system ciphers in
# the case of OpenSSL 1.1.1+ or use our own secure default ciphers.
if ciphers:
context.set_ciphers(ciphers)
context.set_ciphers(ciphers or DEFAULT_CIPHERS)
# Setting the default here, as we may have no ssl module on import
cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs
@@ -283,103 +320,65 @@ def create_urllib3_context(
context.options |= options
if verify_flags is None:
verify_flags = 0
# In Python 3.13+ ssl.create_default_context() sets VERIFY_X509_PARTIAL_CHAIN
# and VERIFY_X509_STRICT so we do the same
if sys.version_info >= (3, 13):
verify_flags |= VERIFY_X509_PARTIAL_CHAIN
verify_flags |= VERIFY_X509_STRICT
context.verify_flags |= verify_flags
# Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is
# necessary for conditional client cert authentication with TLS 1.3.
# The attribute is None for OpenSSL <= 1.1.0 or does not exist when using
# an SSLContext created by pyOpenSSL.
if getattr(context, "post_handshake_auth", None) is not None:
# The attribute is None for OpenSSL <= 1.1.0 or does not exist in older
# versions of Python. We only enable on Python 3.7.4+ or if certificate
# verification is enabled to work around Python issue #37428
# See: https://bugs.python.org/issue37428
if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr(
context, "post_handshake_auth", None
) is not None:
context.post_handshake_auth = True
def disable_check_hostname():
if (
getattr(context, "check_hostname", None) is not None
): # Platform-specific: Python 3.2
# We do our own verification, including fingerprints and alternative
# hostnames. So disable it here
context.check_hostname = False
# The order of the below lines setting verify_mode and check_hostname
# matter due to safe-guards SSLContext has to prevent an SSLContext with
# check_hostname=True, verify_mode=NONE/OPTIONAL.
# We always set 'check_hostname=False' for pyOpenSSL so we rely on our own
# 'ssl.match_hostname()' implementation.
if cert_reqs == ssl.CERT_REQUIRED and not IS_PYOPENSSL:
# check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more
# complex because we don't know whether PROTOCOL_TLS_CLIENT will be used
# or not so we don't know the initial state of the freshly created SSLContext.
if cert_reqs == ssl.CERT_REQUIRED:
context.verify_mode = cert_reqs
context.check_hostname = True
disable_check_hostname()
else:
context.check_hostname = False
disable_check_hostname()
context.verify_mode = cert_reqs
context.hostname_checks_common_name = False
if "SSLKEYLOGFILE" in os.environ:
sslkeylogfile = os.path.expandvars(os.environ.get("SSLKEYLOGFILE"))
else:
sslkeylogfile = None
if sslkeylogfile:
context.keylog_filename = sslkeylogfile
# Enable logging of TLS session keys via defacto standard environment variable
# 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values.
if hasattr(context, "keylog_filename"):
sslkeylogfile = os.environ.get("SSLKEYLOGFILE")
if sslkeylogfile:
context.keylog_filename = sslkeylogfile
return context
@typing.overload
def ssl_wrap_socket(
sock: socket.socket,
keyfile: str | None = ...,
certfile: str | None = ...,
cert_reqs: int | None = ...,
ca_certs: str | None = ...,
server_hostname: str | None = ...,
ssl_version: int | None = ...,
ciphers: str | None = ...,
ssl_context: ssl.SSLContext | None = ...,
ca_cert_dir: str | None = ...,
key_password: str | None = ...,
ca_cert_data: None | str | bytes = ...,
tls_in_tls: typing.Literal[False] = ...,
) -> ssl.SSLSocket: ...
@typing.overload
def ssl_wrap_socket(
sock: socket.socket,
keyfile: str | None = ...,
certfile: str | None = ...,
cert_reqs: int | None = ...,
ca_certs: str | None = ...,
server_hostname: str | None = ...,
ssl_version: int | None = ...,
ciphers: str | None = ...,
ssl_context: ssl.SSLContext | None = ...,
ca_cert_dir: str | None = ...,
key_password: str | None = ...,
ca_cert_data: None | str | bytes = ...,
tls_in_tls: bool = ...,
) -> ssl.SSLSocket | SSLTransportType: ...
def ssl_wrap_socket(
sock: socket.socket,
keyfile: str | None = None,
certfile: str | None = None,
cert_reqs: int | None = None,
ca_certs: str | None = None,
server_hostname: str | None = None,
ssl_version: int | None = None,
ciphers: str | None = None,
ssl_context: ssl.SSLContext | None = None,
ca_cert_dir: str | None = None,
key_password: str | None = None,
ca_cert_data: None | str | bytes = None,
tls_in_tls: bool = False,
) -> ssl.SSLSocket | SSLTransportType:
sock,
keyfile=None,
certfile=None,
cert_reqs=None,
ca_certs=None,
server_hostname=None,
ssl_version=None,
ciphers=None,
ssl_context=None,
ca_cert_dir=None,
key_password=None,
ca_cert_data=None,
tls_in_tls=False,
):
"""
All arguments except for server_hostname, ssl_context, tls_in_tls, ca_cert_data and
ca_cert_dir have the same meaning as they do when using
:func:`ssl.create_default_context`, :meth:`ssl.SSLContext.load_cert_chain`,
:meth:`ssl.SSLContext.set_ciphers` and :meth:`ssl.SSLContext.wrap_socket`.
All arguments except for server_hostname, ssl_context, and ca_cert_dir have
the same meaning as they do when using :func:`ssl.wrap_socket`.
:param server_hostname:
When SNI is supported, the expected hostname of the certificate
@@ -402,18 +401,19 @@ def ssl_wrap_socket(
"""
context = ssl_context
if context is None:
# Note: This branch of code and all the variables in it are only used in tests.
# We should consider deprecating and removing this code.
# Note: This branch of code and all the variables in it are no longer
# used by urllib3 itself. We should consider deprecating and removing
# this code.
context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers)
if ca_certs or ca_cert_dir or ca_cert_data:
try:
context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data)
except OSError as e:
raise SSLError(e) from e
except (IOError, OSError) as e:
raise SSLError(e)
elif ssl_context is None and hasattr(context, "load_default_certs"):
# try to load OS default certs; works well on Windows.
# try to load OS default certs; works well on Windows (require Python3.4+)
context.load_default_certs()
# Attempt to detect if we get the goofy behavior of the
@@ -428,28 +428,57 @@ def ssl_wrap_socket(
else:
context.load_cert_chain(certfile, keyfile, key_password)
context.set_alpn_protocols(ALPN_PROTOCOLS)
try:
if hasattr(context, "set_alpn_protocols"):
context.set_alpn_protocols(ALPN_PROTOCOLS)
except NotImplementedError: # Defensive: in CI, we always have set_alpn_protocols
pass
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
# If we detect server_hostname is an IP address then the SNI
# extension should not be used according to RFC3546 Section 3.1
use_sni_hostname = server_hostname and not is_ipaddress(server_hostname)
# SecureTransport uses server_hostname in certificate verification.
send_sni = (use_sni_hostname and HAS_SNI) or (
IS_SECURETRANSPORT and server_hostname
)
# Do not warn the user if server_hostname is an invalid SNI hostname.
if not HAS_SNI and use_sni_hostname:
warnings.warn(
"An HTTPS request has been made, but the SNI (Server Name "
"Indication) extension to TLS is not available on this platform. "
"This may cause the server to present an incorrect TLS "
"certificate, which can cause validation failures. You can upgrade to "
"a newer version of Python to solve this. For more information, see "
"https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html"
"#ssl-warnings",
SNIMissingWarning,
)
if send_sni:
ssl_sock = _ssl_wrap_socket_impl(
sock, context, tls_in_tls, server_hostname=server_hostname
)
else:
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls)
return ssl_sock
def is_ipaddress(hostname: str | bytes) -> bool:
def is_ipaddress(hostname):
"""Detects whether the hostname given is an IPv4 or IPv6 address.
Also detects IPv6 addresses with Zone IDs.
:param str hostname: Hostname to examine.
:return: True if the hostname is an IP address, False otherwise.
"""
if isinstance(hostname, bytes):
if not six.PY2 and isinstance(hostname, bytes):
# IDN A-label bytes are ASCII compatible.
hostname = hostname.decode("ascii")
return bool(_IPV4_RE.match(hostname) or _BRACELESS_IPV6_ADDRZ_RE.match(hostname))
return bool(IPV4_RE.match(hostname) or BRACELESS_IPV6_ADDRZ_RE.match(hostname))
def _is_key_file_encrypted(key_file: str) -> bool:
def _is_key_file_encrypted(key_file):
"""Detects if a key file is encrypted or not."""
with open(key_file) as f:
with open(key_file, "r") as f:
for line in f:
# Look for Proc-Type: 4,ENCRYPTED
if "ENCRYPTED" in line:
@@ -458,12 +487,7 @@ def _is_key_file_encrypted(key_file: str) -> bool:
return False
def _ssl_wrap_socket_impl(
sock: socket.socket,
ssl_context: ssl.SSLContext,
tls_in_tls: bool,
server_hostname: str | None = None,
) -> ssl.SSLSocket | SSLTransportType:
def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None):
if tls_in_tls:
if not SSLTransport:
# Import error, ssl is not available.
@@ -474,4 +498,7 @@ def _ssl_wrap_socket_impl(
SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context)
return SSLTransport(sock, ssl_context, server_hostname)
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
if server_hostname:
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
else:
return ssl_context.wrap_socket(sock)

Some files were not shown because too many files have changed in this diff Show More