evlli/hass-core
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Generate requirements_* from manifests (#22718)

Browse source 
## Description:
Generate requirements_* from manifests (if present). If not, fallback to the current approach of reading `REQUIREMENTS` from the module attribute. I disabled exploring the children of the `homeassistant.components.*` packages since that will just add a dependency (from the manifest) due to each of the python files in the package. Just having one for the top level package should be sufficient.

**Related issue (if applicable):** relates to #22700 

## Checklist:
  - [x] The code change is tested and works locally.
  - [x] Local tests pass with `tox`. **Your PR cannot be merged unless tests pass**
  - [x] There is no commented out code in this PR.

[ex-requir]: https://github.com/home-assistant/home-assistant/blob/dev/homeassistant/components/keyboard/__init__.py#L14
[ex-import]: https://github.com/home-assistant/home-assistant/blob/dev/homeassistant/components/keyboard/__init__.py#L23


Co-authored-by: Jason Hu <awaregit@gmail.com>
This commit is contained in:
Rohan Kapoor 2019-04-04 21:29:29 -07:00 • committed by Robbie Trencheny
parent 6c5f0b7434
commit d15eedc0fb
4 changed files with 515 additions and 524 deletions
Download patch file Download diff file Expand all files Collapse all files

834
requirements_all.txt
View file

File diff suppressed because it is too large Load diff

110
requirements_test_all.txt
View file

@ -26,19 +26,19 @@ HAP-python==2.4.2
# homeassistant.components.owntracks # homeassistant.components.owntracks
PyNaCl==1.3.0 PyNaCl==1.3.0
# homeassistant.components.rmvtransport.sensor # homeassistant.components.rmvtransport
PyRMVtransport==0.1.3 PyRMVtransport==0.1.3
# homeassistant.components.transport_nsw.sensor # homeassistant.components.transport_nsw
PyTransportNSW==0.1.1 PyTransportNSW==0.1.1
# homeassistant.components.yessssms.notify # homeassistant.components.yessssms
YesssSMS==0.2.3 YesssSMS==0.2.3
# homeassistant.components.ambient_station # homeassistant.components.ambient_station
aioambient==0.2.0 aioambient==0.1.3
# homeassistant.components.automatic.device_tracker # homeassistant.components.automatic
aioautomatic==0.6.5 aioautomatic==0.6.5
# homeassistant.components.aws # homeassistant.components.aws
@ -54,7 +54,7 @@ aiohue==1.9.1
# homeassistant.components.unifi # homeassistant.components.unifi
aiounifi==4 aiounifi==4
# homeassistant.components.apns.notify # homeassistant.components.apns
apns2==0.3.0 apns2==0.3.0
# homeassistant.components.stream # homeassistant.components.stream
@ -66,49 +66,49 @@ axis==19
# homeassistant.components.zha # homeassistant.components.zha
bellows-homeassistant==0.7.2 bellows-homeassistant==0.7.2
# homeassistant.components.caldav.calendar # homeassistant.components.caldav
caldav==0.5.0 caldav==0.5.0
# homeassistant.components.coinmarketcap.sensor # homeassistant.components.coinmarketcap
coinmarketcap==5.0.3 coinmarketcap==5.0.3
# homeassistant.components.ihc # homeassistant.components.ihc
# homeassistant.components.namecheapdns # homeassistant.components.namecheapdns
# homeassistant.components.ohmconnect.sensor # homeassistant.components.ohmconnect
# homeassistant.components.upc_connect.device_tracker # homeassistant.components.upc_connect
defusedxml==0.5.0 defusedxml==0.5.0
# homeassistant.components.dsmr.sensor # homeassistant.components.dsmr
dsmr_parser==0.12 dsmr_parser==0.12
# homeassistant.components.ee_brightbox.device_tracker # homeassistant.components.ee_brightbox
eebrightbox==0.0.4 eebrightbox==0.0.4
# homeassistant.components.emulated_roku # homeassistant.components.emulated_roku
emulated_roku==0.1.8 emulated_roku==0.1.8
# homeassistant.components.season.sensor # homeassistant.components.season
ephem==3.7.6.0 ephem==3.7.6.0
# homeassistant.components.evohome # homeassistant.components.evohome
# homeassistant.components.honeywell.climate # homeassistant.components.honeywell
evohomeclient==0.3.2 evohomeclient==0.3.2
# homeassistant.components.feedreader # homeassistant.components.feedreader
feedparser-homeassistant==5.2.2.dev1 feedparser-homeassistant==5.2.2.dev1
# homeassistant.components.foobot.sensor # homeassistant.components.foobot
foobot_async==0.3.1 foobot_async==0.3.1
# homeassistant.components.google.tts # homeassistant.components.google
gTTS-token==1.1.3 gTTS-token==1.1.3
# homeassistant.components.geo_json_events.geo_location # homeassistant.components.geo_json_events
# homeassistant.components.nsw_rural_fire_service_feed.geo_location # homeassistant.components.nsw_rural_fire_service_feed
# homeassistant.components.usgs_earthquakes_feed.geo_location # homeassistant.components.usgs_earthquakes_feed
geojson_client==0.3 geojson_client==0.3
# homeassistant.components.geo_rss_events.sensor # homeassistant.components.geo_rss_events
georss_generic_client==0.2 georss_generic_client==0.2
# homeassistant.components.ffmpeg # homeassistant.components.ffmpeg
@ -120,13 +120,13 @@ hangups==0.4.6
# homeassistant.components.cloud # homeassistant.components.cloud
hass-nabucasa==0.11 hass-nabucasa==0.11
# homeassistant.components.mqtt.server # homeassistant.components.mqtt
hbmqtt==0.9.4 hbmqtt==0.9.4
# homeassistant.components.jewish_calendar.sensor # homeassistant.components.jewish_calendar
hdate==0.8.7 hdate==0.8.7
# homeassistant.components.workday.binary_sensor # homeassistant.components.workday
holidays==0.9.10 holidays==0.9.10
# homeassistant.components.frontend # homeassistant.components.frontend
@ -139,7 +139,6 @@ homekit[IP]==0.13.0
homematicip==0.10.6 homematicip==0.10.6
# homeassistant.components.influxdb # homeassistant.components.influxdb
# homeassistant.components.influxdb.sensor
influxdb==5.2.0 influxdb==5.2.0
# homeassistant.components.verisure # homeassistant.components.verisure
@ -148,7 +147,7 @@ jsonpath==0.75
# homeassistant.components.dyson # homeassistant.components.dyson
libpurecool==0.5.0 libpurecool==0.5.0
# homeassistant.components.soundtouch.media_player # homeassistant.components.soundtouch
libsoundtouch==0.7.2 libsoundtouch==0.7.2
# homeassistant.components.luftdaten # homeassistant.components.luftdaten
@ -157,38 +156,36 @@ luftdaten==0.3.4
# homeassistant.components.mythicbeastsdns # homeassistant.components.mythicbeastsdns
mbddns==0.1.2 mbddns==0.1.2
# homeassistant.components.mfi.sensor # homeassistant.components.mfi
# homeassistant.components.mfi.switch
mficlient==0.3.0 mficlient==0.3.0
# homeassistant.components.opencv.image_processing # homeassistant.components.opencv
# homeassistant.components.pollen.sensor # homeassistant.components.pollen
# homeassistant.components.tensorflow.image_processing # homeassistant.components.tensorflow
# homeassistant.components.trend.binary_sensor # homeassistant.components.trend
numpy==1.16.2 numpy==1.16.2
# homeassistant.components.mqtt # homeassistant.components.mqtt
# homeassistant.components.shiftr # homeassistant.components.shiftr
paho-mqtt==1.4.0 paho-mqtt==1.4.0
# homeassistant.components.aruba.device_tracker # homeassistant.components.aruba
# homeassistant.components.cisco_ios.device_tracker # homeassistant.components.cisco_ios
# homeassistant.components.pandora.media_player # homeassistant.components.pandora
# homeassistant.components.unifi_direct.device_tracker # homeassistant.components.unifi_direct
pexpect==4.6.0 pexpect==4.6.0
# homeassistant.components.pilight # homeassistant.components.pilight
pilight==0.1.1 pilight==0.1.1
# homeassistant.components.mhz19.sensor # homeassistant.components.mhz19
# homeassistant.components.serial_pm.sensor # homeassistant.components.serial_pm
pmsensor==0.4 pmsensor==0.4
# homeassistant.components.prometheus # homeassistant.components.prometheus
prometheus_client==0.2.0 prometheus_client==0.2.0
# homeassistant.components.pushbullet.notify # homeassistant.components.pushbullet
# homeassistant.components.pushbullet.sensor
pushbullet.py==0.11.0 pushbullet.py==0.11.0
# homeassistant.components.canary # homeassistant.components.canary
@ -197,7 +194,7 @@ py-canary==0.5.0
# homeassistant.components.tplink # homeassistant.components.tplink
pyHS100==0.3.4 pyHS100==0.3.4
# homeassistant.components.blackbird.media_player # homeassistant.components.blackbird
pyblackbird==0.5 pyblackbird==0.5
# homeassistant.components.deconz # homeassistant.components.deconz
@ -215,11 +212,10 @@ pyhomematic==0.1.58
# homeassistant.components.litejet # homeassistant.components.litejet
pylitejet==0.1 pylitejet==0.1
# homeassistant.components.monoprice.media_player # homeassistant.components.monoprice
pymonoprice==0.3 pymonoprice==0.3
# homeassistant.components.nx584.alarm_control_panel # homeassistant.components.nx584
# homeassistant.components.nx584.binary_sensor
pynx584==0.4 pynx584==0.4
# homeassistant.components.openuv # homeassistant.components.openuv
@ -227,7 +223,7 @@ pyopenuv==1.0.9
# homeassistant.auth.mfa_modules.notify # homeassistant.auth.mfa_modules.notify
# homeassistant.auth.mfa_modules.totp # homeassistant.auth.mfa_modules.totp
# homeassistant.components.otp.sensor # homeassistant.components.otp
pyotp==2.2.6 pyotp==2.2.6
# homeassistant.components.ps4 # homeassistant.components.ps4
@ -248,24 +244,23 @@ pysonos==0.0.8
# homeassistant.components.spc # homeassistant.components.spc
pyspcwebgw==0.4.0 pyspcwebgw==0.4.0
# homeassistant.components.darksky.sensor # homeassistant.components.darksky
# homeassistant.components.darksky.weather
python-forecastio==1.4.0 python-forecastio==1.4.0
# homeassistant.components.nest # homeassistant.components.nest
python-nest==4.1.0 python-nest==4.1.0
# homeassistant.components.awair.sensor # homeassistant.components.awair
python_awair==0.0.3 python_awair==0.0.3
# homeassistant.components.tradfri # homeassistant.components.tradfri
pytradfri[async]==6.0.1 pytradfri[async]==6.0.1
# homeassistant.components.unifi.device_tracker # homeassistant.components.unifi
pyunifi==2.16 pyunifi==2.16
# homeassistant.components.html5.notify # homeassistant.components.html5
pywebpush==1.9.2 pywebpush==1.6.0
# homeassistant.components.rainmachine # homeassistant.components.rainmachine
regenmaschine==1.4.0 regenmaschine==1.4.0
@ -279,7 +274,7 @@ rflink==0.0.37
# homeassistant.components.ring # homeassistant.components.ring
ring_doorbell==0.2.3 ring_doorbell==0.2.3
# homeassistant.components.yamaha.media_player # homeassistant.components.yamaha
rxv==0.6.0 rxv==0.6.0
# homeassistant.components.simplisafe # homeassistant.components.simplisafe
@ -291,14 +286,14 @@ sleepyq==0.6
# homeassistant.components.smhi # homeassistant.components.smhi
smhi-pkg==1.0.10 smhi-pkg==1.0.10
# homeassistant.components.honeywell.climate # homeassistant.components.honeywell
somecomfort==0.5.2 somecomfort==0.5.2
# homeassistant.components.recorder # homeassistant.components.recorder
# homeassistant.components.sql.sensor # homeassistant.components.sql
sqlalchemy==1.3.0 sqlalchemy==1.3.0
# homeassistant.components.srp_energy.sensor # homeassistant.components.srp_energy
srpenergy==1.0.6 srpenergy==1.0.6
# homeassistant.components.statsd # homeassistant.components.statsd
@ -307,7 +302,7 @@ statsd==3.2.1
# homeassistant.components.toon # homeassistant.components.toon
toonapilib==3.2.2 toonapilib==3.2.2
# homeassistant.components.uvc.camera # homeassistant.components.uvc
uvcclient==0.11.0 uvcclient==0.11.0
# homeassistant.components.verisure # homeassistant.components.verisure
@ -316,10 +311,9 @@ vsure==1.5.2
# homeassistant.components.vultr # homeassistant.components.vultr
vultr==0.1.2 vultr==0.1.2
# homeassistant.components.panasonic_viera
# homeassistant.components.samsungtv
# homeassistant.components.wake_on_lan # homeassistant.components.wake_on_lan
# homeassistant.components.panasonic_viera.media_player
# homeassistant.components.samsungtv.media_player
# homeassistant.components.wake_on_lan.switch
wakeonlan==1.1.6 wakeonlan==1.1.6
# homeassistant.components.zha # homeassistant.components.zha

73
script/gen_requirements_all.py
View file

@ -7,6 +7,8 @@ import pkgutil
import re import re
import sys import sys
from script.manifest.requirements import gather_requirements_from_manifests
COMMENT_REQUIREMENTS = ( COMMENT_REQUIREMENTS = (
'Adafruit-DHT', 'Adafruit-DHT',
'Adafruit_BBIO', 'Adafruit_BBIO',
@ -213,36 +215,8 @@ def gather_modules():
errors = [] errors = []
for package in sorted( gather_requirements_from_manifests(process_requirements, errors, reqs)
explore_module('homeassistant.components', True) + gather_requirements_from_modules(errors, reqs)
explore_module('homeassistant.scripts', True) +
explore_module('homeassistant.auth', True)):
try:
module = importlib.import_module(package)
except ImportError as err:
for pattern in IGNORE_PACKAGES:
if fnmatch.fnmatch(package, pattern):
break
else:
print("{}: {}".format(package.replace('.', '/') + '.py', err))
errors.append(package)
continue
if not getattr(module, 'REQUIREMENTS', None):
continue
for req in module.REQUIREMENTS:
if req in IGNORE_REQ:
continue
if '://' in req and 'pyharmony' not in req:
errors.append(
"{}[Only pypi dependencies are allowed: {}]".format(
package, req))
if req.partition('==')[1] == '' and req not in IGNORE_PIN:
errors.append(
"{}[Please pin requirement {}, see {}]".format(
package, req, URL_PIN))
reqs.setdefault(req, []).append(package)
for key in reqs: for key in reqs:
reqs[key] = sorted(reqs[key], reqs[key] = sorted(reqs[key],
@ -257,12 +231,47 @@ def gather_modules():
return reqs return reqs
def gather_requirements_from_modules(errors, reqs):
"""Collect the requirements from the modules directly."""
for package in sorted(
explore_module('homeassistant.scripts', True) +
explore_module('homeassistant.auth', True)):
try:
module = importlib.import_module(package)
except ImportError as err:
for pattern in IGNORE_PACKAGES:
if fnmatch.fnmatch(package, pattern):
break
else:
print("{}: {}".format(package.replace('.', '/') + '.py', err))
errors.append(package)
continue
if getattr(module, 'REQUIREMENTS', None):
process_requirements(errors, module.REQUIREMENTS, package, reqs)
def process_requirements(errors, module_requirements, package, reqs):
"""Process all of the requirements."""
for req in module_requirements:
if req in IGNORE_REQ:
continue
if '://' in req:
errors.append(
"{}[Only pypi dependencies are allowed: {}]".format(
package, req))
if req.partition('==')[1] == '' and req not in IGNORE_PIN:
errors.append(
"{}[Please pin requirement {}, see {}]".format(
package, req, URL_PIN))
reqs.setdefault(req, []).append(package)
def generate_requirements_list(reqs): def generate_requirements_list(reqs):
"""Generate a pip file based on requirements.""" """Generate a pip file based on requirements."""
output = [] output = []
for pkg, requirements in sorted(reqs.items(), key=lambda item: item[0]): for pkg, requirements in sorted(reqs.items(), key=lambda item: item[0]):
for req in sorted(requirements, for req in sorted(requirements):
key=lambda name: (len(name.split('.')), name)):
output.append('\n# {}'.format(req)) output.append('\n# {}'.format(req))
if comment_requirement(pkg): if comment_requirement(pkg):

22
script/manifest/requirements.py Normal file
View file

@ -0,0 +1,22 @@
"""Helpers to gather requirements from manifests."""
from .manifest_helper import iter_manifests
def gather_requirements_from_manifests(process_requirements, errors, reqs):
"""Gather all of the requirements from manifests."""
for manifest in iter_manifests():
assert manifest['domain']
if manifest.get('requirements') is None:
errors.append(
'The manifest for component {} is invalid. Please run'
'script/manifest/validate.py'.format(manifest['domain'])
)
continue
process_requirements(
errors,
manifest['requirements'],
'homeassistant.components.{}'.format(manifest['domain']),
reqs
)