diff --git a/CHANGELOG.md b/CHANGELOG.md index fc2e6437..629317eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,9 @@ ### Added (models) -* **ADD** nuclear power plant technology with capacity limits. Capacity limits can be equals to today or be bound by a minimum and maximum capacity to represent an available range in future. In either case, capacities are allocated at a subnational resolution based on linear scaling from current capacity geolocations, using the JRC power plant database (#78). +* **ADD** fully-electrified road transportation (#270). + +* **ADD** nuclear power plant technology with capacity limits. Capacity limits can be equal to today or be bound by a minimum and maximum capacity to represent an available range in future. In either case, capacities are allocated at a subnational resolution based on linear scaling from current capacity geolocations, using the JRC power plant database (#78). ### Added (workflow) diff --git a/Snakefile b/Snakefile index ef6b70e0..2642c118 100644 --- a/Snakefile +++ b/Snakefile @@ -15,12 +15,14 @@ model_template_dir = f"{template_dir}models/" techs_template_dir = f"{model_template_dir}techs/" include: "./rules/shapes.smk" +include: "./rules/data.smk" include: "./rules/wind-and-solar.smk" include: "./rules/biofuels.smk" include: "./rules/hydro.smk" include: "./rules/transmission.smk" include: "./rules/demand.smk" include: "./rules/nuclear.smk" +include: "./rules/transport.smk" include: "./rules/sync.smk" min_version("7.8") localrules: all, clean @@ -35,7 +37,6 @@ ALL_CF_TECHNOLOGIES = [ "rooftop-pv", "rooftop-pv-n", "rooftop-pv-e-w", "rooftop-pv-s-flat", "hydro-run-of-river", "hydro-reservoir" ] -ALL_DEMAND_CARRIERS = ["electricity"] def ensure_lib_folder_is_linked(): if not workflow.conda_prefix: @@ -152,6 +153,7 @@ rule model_template: "interest-rate.yaml", "locations.yaml", "techs/demand/electricity.yaml", + "techs/demand/electrified-transport.yaml", "techs/storage/electricity.yaml", "techs/storage/hydro.yaml", "techs/supply/biofuel.yaml", @@ -167,9 +169,10 @@ rule model_template: "build/models/{{resolution}}/timeseries/supply/capacityfactors-{technology}.csv", technology=ALL_CF_TECHNOLOGIES ), - demand_timeseries_data = expand( - "build/models/{{resolution}}/timeseries/demand/{energy_carrier}.csv", - energy_carrier=ALL_DEMAND_CARRIERS + demand_timeseries_data = ( + "build/models/{resolution}/timeseries/demand/electricity.csv", + "build/models/{resolution}/timeseries/demand/electrified-road-transport.csv", + "build/models/{resolution}/timeseries/demand/road-transport-historic-electrification.csv" ), optional_input_files = lambda wildcards: expand( f"build/models/{wildcards.resolution}/{{input_file}}", diff --git a/config/default.yaml b/config/default.yaml index 2b13bd7d..c4001502 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -13,6 +13,10 @@ data-sources: potentials: https://zenodo.org/record/5112963/files/possibility-for-electricity-autarky.zip entsoe-tyndp: https://2020.entsos-tyndp-scenarios.eu/wp-content/uploads/2020/06/TYNDP-2020-Scenario-Datafile.xlsx.zip jrc-ppdb: https://zenodo.org/record/3574566/files/JRC-PPDB-OPEN.ver1.0.zip + jrc-idees: https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/JRC-IDEES/JRC-IDEES-2015_v1/JRC-IDEES-2015_All_xlsx_{country_code}.zip + eurostat-energy-balance: https://raw.githubusercontent.com/calliope-project/euro-calliope-datasets/feature-sector-coupling/eurostat/nrg_bal_c.tsv.gz # FIXME do not use cached data + swiss-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/gesamtenergiestatistik.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvNzUxOQ==.html + swiss-industry-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/teilstatistiken.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvODc4OA==.html root-directory: . cluster-sync: url: euler.ethz.ch @@ -121,6 +125,28 @@ parameters: proxy: population biofuel-efficiency: 0.45 wind-and-solar-potential-scenario: technical-potential + transport: + future-vehicle-efficiency-percentile: 0.25 # FIXME Why so low? Why not 0.5? + # FIXME road-transport-conversion-factors are redundant, as they are derived using future-vehicle-efficiency-percentile + road-transport-conversion-factors: # MWh / mio km 25th percentile efficiency of all countries in 2015 [@Mantzos:2017] + light-duty-vehicles: 480 + heavy-duty-vehicles: 3248 # based on buses + coaches-and-buses: 3248 + passenger-cars: 324 + motorcycles: 200 # based on passenger car electrical efficiency scaled by relative diesel efficiency + names: + light-duty-vehicles: Light duty vehicles + heavy-duty-vehicles: Heavy duty vehicles + coaches-and-buses: Motor coaches, buses and trolley buses + passenger-cars: Passenger cars + motorcycles: Powered 2-wheelers + fill-missing-values: + ALB: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC'] + BIH: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC'] + MNE: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC'] + SRB: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC'] + NOR: ['SWE', 'DNK'] + CHE: ['DEU', 'AUT', 'FRA', 'ITA'] entsoe-tyndp: scenario: National Trends grid: Reference diff --git a/config/energy-balances/energy-balance-carrier-names.csv b/config/energy-balances/energy-balance-carrier-names.csv new file mode 100644 index 00000000..d03766f4 --- /dev/null +++ b/config/energy-balances/energy-balance-carrier-names.csv @@ -0,0 +1,75 @@ +carrier_code,carrier_name,hh_carrier_name,com_carrier_name,ind_carrier_name,oth_carrier_name +TOTAL,TOTAL,,,, +FF,Fossil fuels,,,, +C0000X0350-0370,Solid fossil fuels,solid_fossil,solid_fossil,solid_fossil,solid_fossil +C0110,Anthracite,,,, +C0121,Coking coal,,,, +C0129,Other bituminous coal,,,, +C0210,Sub-bituminous coal,,,, +C0220,Lignite,,,, +C0311,Coke oven coke,,,, +C0312,Gas coke,,,, +C0320,Patent fuel,,,, +C0330,Brown coal briquettes,,,, +C0340,Coal tar,,,, +C0350-0370,Manufactured gases,,gas,manufactured_gas,gas +C0350,Coke oven gas,,,, +C0360,Gas works gas,,,, +C0371,Blast furnace gas,,,, +C0379,Other recovered gases,,,, +P1000,Peat and peat products,solid_fossil,solid_fossil,solid_fossil,solid_fossil +P1100,Peat,,,, +P1200,Peat products,,,, +S2000,Oil shale and oil sands,solid_fossil,oil,oil,oil +G3000,Natural gas,gas,gas,natural_gas,gas +O4000XBIO,Oil and petroleum products (excluding biofuel portion),oil,oil,oil, +O4100_TOT,Crude oil,,,, +O4200,Natural gas liquids,,,, +O4300,Refinery feedstocks,,,, +O4400X4410,Additives and oxygenates (excluding biofuel portion),,,, +O4500,Other hydrocarbons,,,, +O4610,Refinery gas,,,, +O4620,Ethane,,,, +O4630,Liquefied petroleum gases,,,, +O4640,Naphtha,,,, +O4651,Aviation gasoline,,,, +O4652XR5210B,Motor gasoline (excluding biofuel portion),,,, +O4653,Gasoline-type jet fuel,,,, +O4661XR5230B,Kerosene-type jet fuel (excluding biofuel portion),,,, +O4669,Other kerosene,,,, +O4671XR5220B,Gas oil and diesel oil (excluding biofuel portion),,,, +O4680,Fuel oil,,,, +O4691,White spirit and special boiling point industrial spirits,,,, +O4692,Lubricants,,,, +O4693,Paraffin waxes,,,, +O4694,Petroleum coke,,,, +O4695,Bitumen,,,, +O4699,Other oil products n.e.c.,,,, +RA000,Renewables and biofuels,,,, +RA100,Hydro,,renewable_heat,renewable_heat,renewable_heat +RA200,Geothermal,,renewable_heat,renewable_heat,renewable_heat +RA300,Wind,,renewable_heat,renewable_heat,renewable_heat +RA410,Solar thermal,solar_thermal,renewable_heat,renewable_heat,renewable_heat +RA420,Solar photovoltaic,,renewable_heat,renewable_heat,renewable_heat +RA500,Tide_wave_ocean,,renewable_heat,renewable_heat,renewable_heat +RA600,Ambient heat (heat pumps),ambient_heat,ambient_heat,ambient_heat,ambient_heat +R5110-5150_W6000RI,Primary solid biofuels,biofuel,biofuel,biofuel,biofuel +R5160,Charcoal,,biofuel,biofuel,biofuel +R5210P,Pure biogasoline,,oil,oil,oil +R5210B,Blended biogasoline,,oil,oil,oil +R5220P,Pure biodiesels,,oil,oil,oil +R5220B,Blended biodiesels,,oil,oil,oil +R5230P,Pure bio jet kerosene,,oil,oil,oil +R5230B,Blended bio jet kerosene,,oil,oil,oil +R5290,Other liquid biofuels,,oil,oil,oil +R5300,Biogases,gas,gas,natural_gas,gas +W6100,Industrial waste (non-renewable),,,, +W6210,Renewable municipal waste,,biofuel,biofuel,biofuel +W6220,Non-renewable municipal waste,,,, +W6100_6220,Non-renewable waste,biofuel,biofuel,biofuel,biofuel +N900H,Nuclear heat,,heat,heat,heat +E7000,Electricity,electricity,electricity,electricity, +H8000,Heat,heat,heat,heat,heat +BIOE,Bioenergy,,,, +O4000,,oil,,, +SFF_P1000_S2000,,solid_fossil,,,solid_fossil \ No newline at end of file diff --git a/config/energy-balances/energy-balance-category-names.csv b/config/energy-balances/energy-balance-category-names.csv new file mode 100644 index 00000000..2e42350c --- /dev/null +++ b/config/energy-balances/energy-balance-category-names.csv @@ -0,0 +1,297 @@ +cat_code,top_cat,sub_cat_contribution,sub_cat_1,sub_cat_2,jrc_idees +PPRD,Gross available energy,+,Primary production,, +RCV_RCY,Gross available energy,+,Recovered & recycled products,, +IMP,Gross available energy,+,Imports,, +EXP,Gross available energy,-,Exports,, +STK_CHG,Gross available energy,+,Change in stock,, +GAE,Gross available energy,=,,, +INTMARB,International maritime bunkers,-,,, +GIC,Gross inland consumption,=,,, +INTAVI,International aviation,-,,, +NRGSUP,Total energy supply,=,,, +GIC2020-2030,Gross inland consumption (Europe 2020-2030),=,,, +PEC2020-2030,Primary energy consumption (Europe 2020-2030),=,,, +FEC2020-2030,Final energy consumption (Europe 2020-2030),=,,, +TI_E,Transformation input,=,,, +TI_EHG_E,Transformation input,+,Electricity & heat generation,, +TI_EHG_MAPE_E,Transformation input,+,Electricity & heat generation,Main activity producer electricity only, +TI_EHG_MAPCHP_E,Transformation input,+,Electricity & heat generation,Main activity producer CHP, +TI_EHG_MAPH_E,Transformation input,+,Electricity & heat generation,Main activity producer heat only, +TI_EHG_APE_E,Transformation input,+,Electricity & heat generation,Autoproducer electricity only, +TI_EHG_APCHP_E,Transformation input,+,Electricity & heat generation,Autoproducer CHP , +TI_EHG_APH_E,Transformation input,+,Electricity & heat generation,Autoproducer heat only, +TI_EHG_EDHP,Transformation input,+,Electricity & heat generation,Electrically driven heat pumps, +TI_EHG_EB,Transformation input,+,Electricity & heat generation,Electric boilers, +TI_EHG_EPS,Transformation input,+,Electricity & heat generation,Electricity for pumped storage, +TI_EHG_DHEP,Transformation input,+,Electricity & heat generation,Derived heat for electricity production, +TI_CO_E,Transformation input,+,Coke ovens,, +TI_BF_E,Transformation input,+,Blast furnaces,, +TI_GW_E,Transformation input,+,Gas works,, +TI_RPI_E,Transformation input,+,Refineries & petrochemical industry,, +TI_RPI_RI_E,Transformation input,+,Refineries & petrochemical industry,Refinery intake, +TI_RPI_BPI_E,Transformation input,+,Refineries & petrochemical industry,Backflows from petrochemical industry, +TI_RPI_PT_E,Transformation input,+,Refineries & petrochemical industry,Products transferred, +TI_RPI_IT_E,Transformation input,+,Refineries & petrochemical industry,Interproduct transfers, +TI_RPI_DU_E,Transformation input,+,Refineries & petrochemical industry,Direct use, +TI_RPI_PII_E,Transformation input,+,Refineries & petrochemical industry,Petrochemical industry intake, +TI_PF_E,Transformation input,+,Patent fuel plants,, +TI_BKBPB_E,Transformation input,+,BKB & PB plants,, +TI_CL_E,Transformation input,+,Coal liquefaction plants,, +TI_BNG_E,Transformation input,+,For blended natural gas,, +TI_LBB_E,Transformation input,+,Liquid biofuels blended,, +TI_CPP_E,Transformation input,+,Charcoal production plants,, +TI_GTL_E,Transformation input,+,Gas-to-liquids plants,, +TI_NSP_E,Transformation input,+,Not elsewhere specified ,, +TO,Transformation output,=,,, +TO_EHG,Transformation output,+,Electricity & heat generation,, +TO_EHG_MAPE,Transformation output,+,Electricity & heat generation,Main activity producer electricity only, +TO_EHG_MAPCHP,Transformation output,+,Electricity & heat generation,Main activity producer CHP, +TO_EHG_MAPH,Transformation output,+,Electricity & heat generation,Main activity producer heat only, +TO_EHG_APE,Transformation output,+,Electricity & heat generation,Autoproducer electricity only, +TO_EHG_APCHP,Transformation output,+,Electricity & heat generation,Autoproducer CHP , +TO_EHG_APH,Transformation output,+,Electricity & heat generation,Autoproducer heat only, +TO_EHG_EDHP,Transformation output,+,Electricity & heat generation,Electrically driven heat pumps, +TO_EHG_EB,Transformation output,+,Electricity & heat generation,Electric boilers, +TO_EHG_PH,Transformation output,+,Electricity & heat generation,Pumped hydro, +TO_EHG_OTH,Transformation output,+,Electricity & heat generation,Other sources, +TO_CO,Transformation output,+,Coke ovens,, +TO_BF,Transformation output,+,Blast furnaces,, +TO_GW,Transformation output,+,Gas works,, +TO_RPI,Transformation output,+,Refineries & petrochemical industry,, +TO_RPI_RO,Transformation output,+,Refineries & petrochemical industry,Refinery output, +TO_RPI_BKFLOW,Transformation output,+,Refineries & petrochemical industry,Backflows, +TO_RPI_PT,Transformation output,+,Refineries & petrochemical industry,Products transferred, +TO_RPI_IT,Transformation output,+,Refineries & petrochemical industry,Interproduct transfers, +TO_RPI_PPR,Transformation output,+,Refineries & petrochemical industry,Primary product receipts, +TO_RPI_PIR,Transformation output,+,Refineries & petrochemical industry,Petrochemical industry returns, +TO_PF,Transformation output,+,Patent fuel plants,, +TO_BKBPB,Transformation output,+,BKB & PB plants,, +TO_CL,Transformation output,+,Coal liquefaction plants,, +TO_BNG,Transformation output,+,Blended in natural gas,, +TO_LBB,Transformation output,+,Liquid biofuels blended,, +TO_CPP,Transformation output,+,Charcoal production plants,, +TO_GTL,Transformation output,+,Gas-to-liquids plants,, +TO_NSP,Transformation output,+,Not elsewhere specified ,, +NRG_E,Energy sector,=,,, +NRG_EHG_E,Energy sector,+,Own use in electricity & heat generation,, +NRG_CM_E,Energy sector,+,Coal mines,, +NRG_OIL_NG_E,Energy sector,+,Oil & natural gas extraction plants,, +NRG_PF_E,Energy sector,+,Patent fuel plants,, +NRG_CO_E,Energy sector,+,Coke ovens,, +NRG_BKBPB_E,Energy sector,+,BKB & PB plants,, +NRG_GW_E,Energy sector,+,Gas works,, +NRG_BF_E,Energy sector,+,Blast furnaces,, +NRG_PR_E,Energy sector,+,Petroleum refineries (oil refineries),, +NRG_NI_E,Energy sector,+,Nuclear industry,, +NRG_CL_E,Energy sector,+,Coal liquefaction plants,, +NRG_LNG_E,Energy sector,+,Liquefaction & regasification plants (LNG),, +NRG_BIOG_E,Energy sector,+,Gasification plants for biogas,, +NRG_GTL_E,Energy sector,+,Gas-to-liquids (GTL) plants,, +NRG_CPP_E,Energy sector,+,Charcoal production plants,, +NRG_NSP_E,Energy sector,+,Not elsewhere specified (energy),, +DL,Distribution losses,=,,, +AFC,Available for final consumption,=,,, +FC_NE,Final non-energy consumption,=,,, +TI_NRG_FC_IND_NE,Final non-energy consumption,+,Non-energy use industry/transformation/energy,, +TI_NE,Final non-energy consumption,+,Non-energy use industry/transformation/energy,Non-energy use in transformation sector, +NRG_NE,Final non-energy consumption,+,Non-energy use industry/transformation/energy,Non-energy use in energy sector, +FC_IND_NE,Final non-energy consumption,+,Non-energy use industry/transformation/energy,Non-energy use in industry sector,Chemicals Industry +FC_TRA_NE,Final non-energy consumption,+,Non-energy use in transport sector,, +FC_OTH_NE,Final non-energy consumption,+,Non-energy use in other sectors,, +FC_E,Final energy consumption,=,,, +FC_IND_E,Final energy consumption,+,Industry sector,, +FC_IND_IS_E,Final energy consumption,+,Industry sector,Iron & steel,Iron and steel +FC_IND_CPC_E,Final energy consumption,+,Industry sector,Chemical & petrochemical,Chemicals Industry +FC_IND_NFM_E,Final energy consumption,+,Industry sector,Non-ferrous metals,Non Ferrous Metals +FC_IND_NMM_E,Final energy consumption,+,Industry sector,Non-metallic minerals,Non-metallic mineral products +FC_IND_TE_E,Final energy consumption,+,Industry sector,Transport equipment,Transport Equipment +FC_IND_MAC_E,Final energy consumption,+,Industry sector,Machinery,Machinery Equipment +FC_IND_MQ_E,Final energy consumption,+,Industry sector,Mining & quarrying,Other Industrial Sectors +FC_IND_FBT_E,Final energy consumption,+,Industry sector,"Food, beverages & tobacco","Food, beverages and tobacco" +FC_IND_PPP_E,Final energy consumption,+,Industry sector,"Paper, pulp & printing","Pulp, paper and printing" +FC_IND_WP_E,Final energy consumption,+,Industry sector,Wood & wood products,Wood and wood products +FC_IND_CON_E,Final energy consumption,+,Industry sector,Construction,Other Industrial Sectors +FC_IND_TL_E,Final energy consumption,+,Industry sector,Textile & leather,Textiles and leather +FC_IND_NSP_E,Final energy consumption,+,Industry sector,Not elsewhere specified (industry),Other Industrial Sectors +FC_TRA_E,Final energy consumption,+,Transport sector,, +FC_TRA_RAIL_E,Final energy consumption,+,Transport sector,Rail, +FC_TRA_ROAD_E,Final energy consumption,+,Transport sector,Road, +FC_TRA_DAVI_E,Final energy consumption,+,Transport sector,Domestic aviation, +FC_TRA_DNAVI_E,Final energy consumption,+,Transport sector,Domestic navigation, +FC_TRA_PIPE_E,Final energy consumption,+,Transport sector,Pipeline transport, +FC_TRA_NSP_E,Final energy consumption,+,Transport sector,Not elsewhere specified (transport), +FC_OTH_E,Final energy consumption,+,Other sectors,, +FC_OTH_CP_E,Final energy consumption,+,Other sectors,Commercial & public services, +FC_OTH_HH_E,Final energy consumption,+,Other sectors,Households, +FC_OTH_AF_E,Final energy consumption,+,Other sectors,Agriculture & forestry, +FC_OTH_FISH_E,Final energy consumption,+,Other sectors,Fishing, +FC_OTH_NSP_E,Final energy consumption,+,Other sectors,Not elsewhere specified (other), +TOTX4_MEMO,Total (excluding LULUCF and memo items),,,, +TOTX4_MEMONIA,"Total (excluding LULUCF and memo items, including international aviation)",,,, +TOTX4_MEMONIT,"Total (excluding LULUCF and memo items, including international transport)",,,, +CRF1,Energy,,,, +CRF1A,Energy,,Fuel combustion - sectoral approach,, +CRF1A1,Energy,,Fuel combustion in energy industries,, +CRF1A1A,Energy,,Fuel combustion in energy industries,Fuel combustion in public electricity and heat production, +CRF1A1B,Energy,,Fuel combustion in energy industries,Fuel combustion in petroleum refining, +CRF1A1C,Energy,,Fuel combustion in energy industries,Fuel combustion in manufacture of solid fuels and other energy industries, +CRF1A2,Energy,,Fuel combustion in manufacturing industries and construction,, +CRF1A2A,Energy,,Fuel combustion in manufacturing industries and construction,Fuel combustion in manufacture of iron and steel, +CRF1A2B,Energy,,Fuel combustion in manufacturing industries and construction,Fuel combustion in manufacture of non-ferrous metals, +CRF1A2C,Energy,,Fuel combustion in manufacturing industries and construction,Fuel combustion in manufacture of chemicals, +CRF1A2D,Energy,,Fuel combustion in manufacturing industries and construction,"Fuel combustion in manufacture of pulp, paper and printing", +CRF1A2E,Energy,,Fuel combustion in manufacturing industries and construction,"Fuel combustion in manufacture of food, beverages and tobacco", +CRF1A2F,Energy,,Fuel combustion in manufacturing industries and construction,Fuel combustion in manufacture of non-metallic mineral products, +CRF1A2G,Energy,,Fuel combustion in manufacturing industries and construction,Fuel combustion in other manufacturing industries and construction, +CRF1A3,Energy,,Fuel combustion in transport,, +CRF1A3A,Energy,,Fuel combustion in transport,Fuel combustion in domestic aviation, +CRF1A3B,Energy,,Fuel combustion in transport,Fuel combustion in road transport, +CRF1A3B1,Energy,,Fuel combustion in transport,Fuel combustion in cars, +CRF1A3B2,Energy,,Fuel combustion in transport,Fuel combustion in light duty trucks, +CRF1A3B3,Energy,,Fuel combustion in transport,Fuel combustion in heavy duty trucks and buses, +CRF1A3B4,Energy,,Fuel combustion in transport,Fuel combustion in motorcycles, +CRF1A3B5,Energy,,Fuel combustion in transport,Fuel combustion in other road transportation, +CRF1A3C,Energy,,Fuel combustion in transport,Fuel combustion in railways, +CRF1A3D,Energy,,Fuel combustion in transport,Fuel combustion in domestic navigation, +CRF1A3E,Energy,,Fuel combustion in transport,Fuel combustion in other transport, +CRF1A4,Energy,,Other fuel combustion sectors,, +CRF1A4A,Energy,,Other fuel combustion sectors,Fuel combustion in commercial and institutional sector, +CRF1A4B,Energy,,Other fuel combustion sectors,Fuel combustion by households, +CRF1A4C,Energy,,Other fuel combustion sectors,"Fuel combustion in agriculture, forestry and fishing", +CRF1A5,Energy,,Other fuel combustion sectors n.e.c.,, +CRF1A5A,Energy,,Other fuel combustion sectors n.e.c.,Stationary fuel combustion sectors n.e.c., +CRF1A5B,Energy,,Other fuel combustion sectors n.e.c.,Mobile fuel combustion sectors n.e.c., +CRF1B,Energy,,Fuels - fugitive emissions,, +CRF1B1,Energy,,Fuels - fugitive emissions,Solid fuels - fugitive emissions, +CRF1B2,Energy,,Fuels - fugitive emissions,"Oil, natural gas and other energy production - fugitive emissions", +CRF1C,Energy,,Transport and storage of CO2 (memo item),, +CRF1D1,Energy,,International bunkers (memo item),, +CRF1D1A,Energy,,International bunkers (memo item),International aviation (memo item), +CRF1D1B,Energy,,International bunkers (memo item),International navigation (memo item), +CRF1D2,Energy,,International bunkers (memo item),Multilateral operations (memo item), +CRF1D3,Energy,,International bunkers (memo item),Biomass - CO2 emissions (memo item), +CRF2,Industrial processes and product use,,,, +CRF2A,Industrial processes and product use,,Mineral industry,, +CRF2A1,Industrial processes and product use,,Mineral industry,Cement production, +CRF2A2,Industrial processes and product use,,Mineral industry,Lime production, +CRF2A3,Industrial processes and product use,,Mineral industry,Glass production, +CRF2A4,Industrial processes and product use,,Mineral industry,Other process uses of carbonates, +CRF2B,Industrial processes and product use,,Chemical industry,, +CRF2B1,Industrial processes and product use,,Chemical industry,Ammonia production, +CRF2B2,Industrial processes and product use,,Chemical industry,Nitric acid production, +CRF2B3,Industrial processes and product use,,Chemical industry,Adipic acid production, +CRF2B4,Industrial processes and product use,,Chemical industry,"Caprolactam, glyoxal and glyoxylic acid production", +CRF2B5,Industrial processes and product use,,Chemical industry,Carbide production, +CRF2B6,Industrial processes and product use,,Chemical industry,Titanium dioxide production, +CRF2B7,Industrial processes and product use,,Chemical industry,Soda ash production, +CRF2B8,Industrial processes and product use,,Chemical industry,Petrochemical and carbon black production, +CRF2B9,Industrial processes and product use,,Chemical industry,Fluorochemical production, +CRF2B10,Industrial processes and product use,,Chemical industry,Other chemical industry, +CRF2C,Industrial processes and product use,,Metal industry,, +CRF2C1,Industrial processes and product use,,Metal industry,Iron and steel production, +CRF2C2,Industrial processes and product use,,Metal industry,Ferroalloys production, +CRF2C3,Industrial processes and product use,,Metal industry,Aluminium production, +CRF2C4,Industrial processes and product use,,Metal industry,Magnesium production, +CRF2C5,Industrial processes and product use,,Metal industry,Lead production, +CRF2C6,Industrial processes and product use,,Metal industry,Zinc production, +CRF2C7,Industrial processes and product use,,Metal industry,Other metal industry, +CRF2D,Industrial processes and product use,,Non-energy products from fuels and solvent use,, +CRF2D1,Industrial processes and product use,,Non-energy products from fuels and solvent use,Lubricant use, +CRF2D2,Industrial processes and product use,,Non-energy products from fuels and solvent use,Paraffin wax use, +CRF2D3,Industrial processes and product use,,Non-energy products from fuels and solvent use,Other non-energy product use, +CRF2E,Industrial processes and product use,,Electronics industry,, +CRF2E1,Industrial processes and product use,,Electronics industry,Integrated circuit or semiconductor production, +CRF2E2,Industrial processes and product use,,Electronics industry,TFT flat panel display production, +CRF2E3,Industrial processes and product use,,Electronics industry,Photovoltaic, +CRF2E4,Industrial processes and product use,,Electronics industry,Heat transfer fluid, +CRF2E5,Industrial processes and product use,,Electronics industry,Other electronics industry, +CRF2F,Industrial processes and product use,,Product uses as substitutes for ozone depleting substances,, +CRF2F1,Industrial processes and product use,,Product uses as substitutes for ozone depleting substances,Refrigeration and air conditioning, +CRF2F2,Industrial processes and product use,,Product uses as substitutes for ozone depleting substances,Foam blowing agent use, +CRF2F3,Industrial processes and product use,,Product uses as substitutes for ozone depleting substances,Fire protection, +CRF2F4,Industrial processes and product use,,Product uses as substitutes for ozone depleting substances,Aerosol use, +CRF2F5,Industrial processes and product use,,Product uses as substitutes for ozone depleting substances,Solvent use, +CRF2F6,Industrial processes and product use,,Product uses as substitutes for ozone depleting substances,Other applications of substitutes for ozone depleting substances, +CRF2G,Industrial processes and product use,,Other product manufacture and use,, +CRF2H,Industrial processes and product use,,Other industrial process and product use,, +CRF3,Agriculture,,,, +CRF31,Agriculture,,Livestock,, +CRF3A,Agriculture,,Enteric fermentation,, +CRF3A1,Agriculture,,Enteric fermentation,Enteric fermentation of cattle, +CRF3A2,Agriculture,,Enteric fermentation,Enteric fermentation of sheep, +CRF3A3,Agriculture,,Enteric fermentation,Enteric fermentation of swine, +CRF3A4,Agriculture,,Enteric fermentation,Enteric fermentation of other livestock, +CRF3B,Agriculture,,Manure management,, +CRF3B1,Agriculture,,Manure management,Cattle manure management, +CRF3B2,Agriculture,,Manure management,Sheep manure management, +CRF3B3,Agriculture,,Manure management,Swine manure management, +CRF3B4,Agriculture,,Manure management,Other livestock manure management, +CRF3B5,Agriculture,,Manure management,Manure management - indirect N2O emissions, +CRF3C,Agriculture,,Rice cultivation,, +CRF3C1,Agriculture,,Rice cultivation,Irrigated rice cultivation, +CRF3C2,Agriculture,,Rice cultivation,Rainfed rice cultivation, +CRF3C3,Agriculture,,Rice cultivation,Deep water rice cultivation, +CRF3C4,Agriculture,,Rice cultivation,Other rice cultivation, +CRF3D,Agriculture,,Managed agricultural soils,, +CRF3D1,Agriculture,,Managed agricultural soils,Managed agricultural soils - direct N2O emissions, +CRF3D2,Agriculture,,Managed agricultural soils,Managed agricultural soils - indirect N2O emissions, +CRF3E,Agriculture,,Prescribed burning of savannas,, +CRF3F,Agriculture,,Field burning of agricultural residues,, +CRF3F1,Agriculture,,Field burning of agricultural residues,Field burning of cereals residues, +CRF3F2,Agriculture,,Field burning of agricultural residues,Field burning of pulses residues, +CRF3F3,Agriculture,,Field burning of agricultural residues,Field burning of tubers and roots residues, +CRF3F4,Agriculture,,Field burning of agricultural residues,Field burning of sugar cane residues, +CRF3F5,Agriculture,,Field burning of agricultural residues,Field burning of other agricultural residues, +CRF3G,Agriculture,,Liming,, +CRF3H,Agriculture,,Urea application,, +CRF3I,Agriculture,,Other carbon-containing fertilizers,, +CRF3J,Agriculture,,Other agriculture,, +CRF4,"Land use, land use change, and forestry (LULUCF)",,,, +CRF4A,"Land use, land use change, and forestry (LULUCF)",,Forest land,, +CRF4A0,"Land use, land use change, and forestry (LULUCF)",,Forest land,Drainage and rewetting and other management of organic and mineral soils related to forest land - emissions and removals, +CRF4A1,"Land use, land use change, and forestry (LULUCF)",,Forest land,Unconverted forest land, +CRF4A2,"Land use, land use change, and forestry (LULUCF)",,Forest land,Land converted to forest land, +CRF4B,"Land use, land use change, and forestry (LULUCF)",,Cropland,, +CRF4B0,"Land use, land use change, and forestry (LULUCF)",,Cropland,Drainage and rewetting and other management of organic and mineral soils related to cropland - emissions and removals, +CRF4B1,"Land use, land use change, and forestry (LULUCF)",,Cropland,Unconverted cropland, +CRF4B2,"Land use, land use change, and forestry (LULUCF)",,Cropland,Land converted to cropland, +CRF4C,"Land use, land use change, and forestry (LULUCF)",,Grassland,, +CRF4C0,"Land use, land use change, and forestry (LULUCF)",,Grassland,Drainage and rewetting and other management of organic and mineral soils related to grassland - emissions and removals, +CRF4C1,"Land use, land use change, and forestry (LULUCF)",,Grassland,Unconverted grassland, +CRF4C2,"Land use, land use change, and forestry (LULUCF)",,Grassland,Land converted to grassland, +CRF4D,"Land use, land use change, and forestry (LULUCF)",,Wetlands,, +CRF4D0,"Land use, land use change, and forestry (LULUCF)",,Wetlands,Drainage and rewetting and other management of organic and mineral soils related to wetlands - emissions and removals, +CRF4D1,"Land use, land use change, and forestry (LULUCF)",,Wetlands,Unconverted wetlands, +CRF4D2,"Land use, land use change, and forestry (LULUCF)",,Wetlands,Land converted to wetlands, +CRF4E,"Land use, land use change, and forestry (LULUCF)",,Settlements,, +CRF4E0,"Land use, land use change, and forestry (LULUCF)",,Settlements,Biomass burning in settlements, +CRF4E1,"Land use, land use change, and forestry (LULUCF)",,Settlements,Unconverted settlements, +CRF4E2,"Land use, land use change, and forestry (LULUCF)",,Settlements,Land converted to settlements, +CRF4F,"Land use, land use change, and forestry (LULUCF)",,Other land,, +CRF4F2,"Land use, land use change, and forestry (LULUCF)",,Other land,Land converted to other land, +CRF4F3,"Land use, land use change, and forestry (LULUCF)",,Other land,Nitrogen mineralization and immobilization in other land - direct N2O emissions, +CRF4F4,"Land use, land use change, and forestry (LULUCF)",,Other land,Biomass burning on other land, +CRF4G,"Land use, land use change, and forestry (LULUCF)",,Harvested wood products,, +CRF4H,"Land use, land use change, and forestry (LULUCF)",,"Other land use, land use change, and forestry",, +CRF4Z,"Land use, land use change, and forestry (LULUCF)",,Managed soils - indirect N2O emissions,, +CRF5,Waste management,,,, +CRF5A,Waste management,,Solid waste disposal,, +CRF5A1,Waste management,,Solid waste disposal,Managed waste disposal sites, +CRF5A2,Waste management,,Solid waste disposal,Unmanaged waste disposal sites, +CRF5A3,Waste management,,Solid waste disposal,Uncategorized waste disposal sites, +CRF5B,Waste management,,Biological treatment of solid waste,, +CRF5B1,Waste management,,Biological treatment of solid waste,Waste composting, +CRF5B2,Waste management,,Biological treatment of solid waste,Anaerobic digestion at biogas facilities, +CRF5C,Waste management,,Incineration and open burning of waste,, +CRF5C1,Waste management,,Incineration and open burning of waste,Waste incineration, +CRF5C2,Waste management,,Incineration and open burning of waste,Open burning of waste, +CRF5D,Waste management,,Wastewater treatment and discharge,, +CRF5D1,Waste management,,Wastewater treatment and discharge,Domestic wastewater, +CRF5D2,Waste management,,Wastewater treatment and discharge,Industrial wastewater, +CRF5D3,Waste management,,Wastewater treatment and discharge,Other wastewater, +CRF5E,Waste management,,Other disposal,, +CRF5F1,Waste management,,Long-term storage of carbon in waste disposal sites,, +CRF5F2,Waste management,,Annual change in total long-term carbon storage,, +CRF5F3,Waste management,,Annual change in total long-term carbon storage in harvested wood products HWP waste,, +CRF6,Other sectors,,,, +CRF_INDCO2,Indirect CO2,,,, \ No newline at end of file diff --git a/config/eurostat/carrier-names.csv b/config/eurostat/carrier-names.csv new file mode 100644 index 00000000..a2203e3b --- /dev/null +++ b/config/eurostat/carrier-names.csv @@ -0,0 +1,75 @@ +carrier_code,carrier_name,household_carrier_name,commercial_carrier_name,industry_carrier_name,other-commercial_carrier_name,aviation_carrier_name,other-aviation_carrier_name,marine_carrier_name,other-marine_carrier_name,road-transport_carrier_name,other-road-transport_carrier_name,rail_carrier_name +TOTAL,TOTAL,,,,,,,,,,, +FF,Fossil fuels,,,,,,,,,,, +C0000X0350-0370,Solid fossil fuels,solid_fossil,solid_fossil,solid_fossil,solid_fossil,,,,,,, +C0110,Anthracite,,,,,,,,,,, +C0121,Coking coal,,,,,,,,,,, +C0129,Other bituminous coal,,,,,,,,,,, +C0210,Sub-bituminous coal,,,,,,,,,,, +C0220,Lignite,,,,,,,,,,, +C0311,Coke oven coke,,,,,,,,,,, +C0312,Gas coke,,,,,,,,,,, +C0320,Patent fuel,,,,,,,,,,, +C0330,Brown coal briquettes,,,,,,,,,,, +C0340,Coal tar,,,,,,,,,,, +C0350-0370,Manufactured gases,,gas,gas,gas,,,,,,, +C0350,Coke oven gas,,,,,,,,,,, +C0360,Gas works gas,,,,,,,,,,, +C0371,Blast furnace gas,,,,,,,,,,, +C0379,Other recovered gases,,,,,,,,,,, +P1000,Peat and peat products,solid_fossil,solid_fossil,solid_fossil,solid_fossil,,,,,,, +P1100,Peat,,,,,,,,,,, +P1200,Peat products,,,,,,,,,,, +S2000,Oil shale and oil sands,oil,oil,oil,oil,,,,,,, +G3000,Natural gas,gas,gas,gas,gas,,,,,natural_gas,,natural_gas +O4000XBIO,Oil and petroleum products (excluding biofuel portion),oil,oil,oil,,jet_fuel,,diesel,diesel,,, +O4100_TOT,Crude oil,,,,,,,,,,, +O4200,Natural gas liquids,,,,,,,,,,, +O4300,Refinery feedstocks,,,,,,,,,,, +O4400X4410,Additives and oxygenates (excluding biofuel portion),,,,,,,,,,, +O4500,Other hydrocarbons,,,,,,,,,,, +O4610,Refinery gas,,,,,,,,,,, +O4620,Ethane,,,,,,,,,,, +O4630,Liquefied petroleum gases,,,,,,,,,lpg,lpg,lpg +O4640,Naphtha,,,,,,,,,,, +O4651,Aviation gasoline,,,,,,jet_fuel,,,,, +O4652XR5210B,Motor gasoline (excluding biofuel portion),,,,,,,,,petrol,petrol,petrol +O4653,Gasoline-type jet fuel,,,,,,jet_fuel,,,,, +O4661XR5230B,Kerosene-type jet fuel (excluding biofuel portion),,,,,,jet_fuel,,,,, +O4669,Other kerosene,,,,,,jet_fuel,,,,, +O4671XR5220B,Gas oil and diesel oil (excluding biofuel portion),,,,,,,,,diesel,diesel,diesel +O4680,Fuel oil,,,,,,,,,,, +O4691,White spirit and special boiling point industrial spirits,,,,,,,,,,, +O4692,Lubricants,,,,,,,,,,, +O4693,Paraffin waxes,,,,,,,,,,, +O4694,Petroleum coke,,,,,,,,,,, +O4695,Bitumen,,,,,,,,,,, +O4699,Other oil products n.e.c.,,,,,,,,,,, +RA000,Renewables and biofuels,,,,,jet_fuel,,diesel,diesel,,, +RA100,Hydro,,renewable_heat,renewable_heat,renewable_heat,,,,,,, +RA200,Geothermal,,renewable_heat,renewable_heat,renewable_heat,,,,,,, +RA300,Wind,,renewable_heat,renewable_heat,renewable_heat,,,,,,, +RA410,Solar thermal,renewable_heat,renewable_heat,renewable_heat,renewable_heat,,,,,,, +RA420,Solar photovoltaic,,renewable_heat,renewable_heat,renewable_heat,,,,,,, +RA500,Tide_wave_ocean,,renewable_heat,renewable_heat,renewable_heat,,,,,,, +RA600,Ambient heat (heat pumps),ambient_heat,ambient_heat,ambient_heat,ambient_heat,,,,,,, +R5110-5150_W6000RI,Primary solid biofuels,biofuel,biofuel,biofuel,biofuel,,,,,,, +R5160,Charcoal,,biofuel,biofuel,biofuel,,,,,,, +R5210P,Pure biogasoline,oil,oil,oil,,,,,,biofuels,biofuels,biofuels +R5210B,Blended biogasoline,oil,oil,oil,,,,,,biofuels,biofuels,biofuels +R5220P,Pure biodiesels,oil,oil,oil,,,,,,biofuels,biofuels,biofuels +R5220B,Blended biodiesels,oil,oil,oil,,,,,,biofuels,biofuels,biofuels +R5230P,Pure bio jet kerosene,,,,,,jet_fuel,,,,, +R5230B,Blended bio jet kerosene,,,,,,jet_fuel,,,,, +R5290,Other liquid biofuels,,oil,oil,oil,,,,,biofuels,,biofuels +R5300,Biogases,gas,gas,gas,gas,,,,,,, +W6100,Industrial waste (non-renewable),,,,,,,,,,, +W6210,Renewable municipal waste,,biofuel,biofuel,biofuel,,,,,,, +W6220,Non-renewable municipal waste,,,,,,,,,,, +W6100_6220,Non-renewable waste,biofuel,biofuel,biofuel,biofuel,,,,,,, +N900H,Nuclear heat,,heat,heat,heat,,,,,,, +E7000,Electricity,electricity,electricity,electricity,,,,,,electricity,,electricity +H8000,Heat,heat,heat,heat,heat,,,,,,, +BIOE,Bioenergy,,,,,,,,,,, +O4000,,oil,oil,,,,,,,,, +SFF_P1000_S2000,,solid_fossil,solid_fossil,,solid_fossil,,,,,,, diff --git a/config/minimal.yaml b/config/minimal.yaml index 9716856c..7367dc4a 100644 --- a/config/minimal.yaml +++ b/config/minimal.yaml @@ -8,6 +8,9 @@ sea-connections: - [GBR, IRL] regional: # Source: https://www.entsoe.eu/data/map/ - [GBR.4_1, IRL.17_1] # Wales and Meath +parameters: + transport: + fill-missing-values: {} scope: spatial: countries: @@ -31,4 +34,3 @@ shapes: # This config must be consistent with data from https://doi.org/10.5281/ regional: Ireland: gadm1 United Kingdom: gadm1 - diff --git a/config/schema.yaml b/config/schema.yaml index 75d69871..4d5f52e6 100644 --- a/config/schema.yaml +++ b/config/schema.yaml @@ -1,4 +1,4 @@ -$schema: http://json-schema.org/draft-07/schema# +$schema: https://json-schema.org/draft/2020-12/schema description: Below you can find a complete enumeration of all configuration parameters of Euro-Calliope's workflow including short descriptions and datatypes. To learn where to find default values and how to change the parameter values for your model builds, head over to the Customisation section of the workflow documentation. definitions: feedstock: @@ -16,6 +16,7 @@ definitions: enum: [farmland, forest, population] required: ["include", "id", "proxy"] type: object +additionalProperties: false properties: email: type: string @@ -25,6 +26,7 @@ properties: data-sources: type: object description: Paths and URLs to datasets. + additionalProperties: false properties: biofuel-potentials-and-costs: type: string @@ -77,12 +79,29 @@ properties: type: string pattern: ^(https?|http?):\/\/.+ description: Web address of JRC power plant database. + jrc-idees: + type: string + pattern: ^(https?|http?):\/\/.+ + description: Web address of JRC IDEES data. + eurostat-energy-balance: + type: string + pattern: ^(https?|http?):\/\/.+ + description: Web address of Eurostat energy balance data. + swiss-energy-balance: + type: string + pattern: ^(https?|http?):\/\/.+ + description: Web address of Swiss energy balance data. + swiss-industry-energy-balance: + type: string + pattern: ^(https?|http?):\/\/.+ + description: Web address of Swiss industry energy balance data. root-directory: type: string description: Path to the root directory of euro-calliope containing scripts and template folders. cluster-sync: type: object description: Configuration for the "work local, build on remote" workflow. + additionalProperties: false properties: url: type: string @@ -102,6 +121,7 @@ properties: scaling-factors: type: object description: Factors to scale the optimisation problem to decrease its numerical range. + additionalProperties: false properties: power: type: number @@ -115,10 +135,12 @@ properties: parameters: type: object description: Parameter values of the models. + additionalProperties: false properties: maximum-installable-power-density: type: object description: Installable capacity density in MW/km² (not annual energy yield). + additionalProperties: false properties: pv-on-tilted-roofs: type: number @@ -139,6 +161,7 @@ properties: roof-share: type: object description: Share of roofs by orientation (should add up to 1) + additionalProperties: false properties: E: type: number @@ -168,6 +191,7 @@ properties: jrc-biofuel: type: object description: Biofuel potential under different scenarios, to select specific data from [@Ruiz:2019] + additionalProperties: false properties: scenario: type: string @@ -228,9 +252,72 @@ properties: type: string description: Scenario defining the amount of surfaces eligible for solar and wind power from [@Trondle:2019]. enum: ["technical-potential", "technical-social-potential"] + transport: + type: object + description: Parameters for transport sector model. + additionalProperties: false + properties: + future-vehicle-efficiency-percentile: + type: number + minimum: 0 + maximum: 1 + description: Percentile of vehicle efficiency across countries. + road-transport-conversion-factors: + type: object + description: Electricity required per distance travelled (MWh / million km). + additionalProperties: false + properties: + light-duty-vehicles: + type: number + description: Light duty vehicles / trucks. + heavy-duty-vehicles: + type: number + description: Heavy duty vehicles / trucks. + coaches-and-buses: + type: number + description: Coaches and buses. + passenger-cars: + type: number + description: Passenger cars. + motorcycles: + type: number + description: Motorcylces. + names: + type: object + description: Names of vehicle types in JRC-IDEES. + additionalProperties: false + properties: + light-duty-vehicles: + type: string + description: JRC-IDEES name of light-duty-vehicles. + heavy-duty-vehicles: + type: string + description: JRC-IDEES name of heavy-duty-vehicles. + coaches-and-buses: + type: string + description: JRC-IDEES name of coaches-and-buses. + passenger-cars: + type: string + description: JRC-IDEES name of passenger-cars. + motorcycles: + type: string + description: JRC-IDEES name of motorcycles. + fill-missing-values: + description: Fill missing values in annual transport demand. + type: object + additionalProperties: false + patternProperties: + "^[A-Z][A-Z][A-Z]$": + type: array + description: Country to fill missing values for. + items: + type: string + pattern: ^[A-Z][A-Z][A-Z]$ + description: Country to fill missing values from. entsoe-tyndp: type: object description: Parameters to define scenario choice for data accessed from the ENTSO-E ten-year network development plan 2020. For more information, see https://2020.entsos-tyndp-scenarios.eu/ + additionalProperties: false properties: scenario: type: string @@ -260,6 +347,7 @@ properties: scope: type: object description: Spatial and temporal scope bounding the models. + additionalProperties: false properties: spatial: type: object @@ -310,6 +398,7 @@ properties: bounds: type: object description: Bounding box describing the spatial extent of geographic datasets. + additionalProperties: false properties: x_min: type: number @@ -334,6 +423,7 @@ properties: temporal: type: object description: The temporal scope of the models. Available range of 2010 - 2016 is set by electricity load data availability (lower bound) and capacity factor data availability (upper bound). + additionalProperties: false properties: first-year: type: integer @@ -349,6 +439,7 @@ properties: quality-control: type: object description: Parameters controlling the data preprocessing steps. + additionalProperties: false properties: hydro: type: object @@ -412,16 +503,19 @@ properties: minimum: 0 maximum: 1 - capacity_factors: + capacity-factors: type: object description: Parameters related to average or time-dependent capacity factors of renewables. + additionalProperties: false properties: min: type: number description: Set smaller values in time series to 0; this helps numerics in the LP. + minimum: 0 max: type: number description: Cap larger values in time series; this helps numerics in the LP (hydro reservoir can have > 1). + minimum: 0 average: type: object description: Average estimation used to transform annual fixed to variable costs. @@ -429,15 +523,19 @@ properties: pv: type: number description: Average annual PV capacity factor + minimum: 0 onshore: type: number description: Average annual onshore wind capacity factor + minimum: 0 offshore: type: number description: Average annual offshore wind capacity factor + minimum: 0 ror: type: number description: Average annual run-of-river hydro capacity factor + minimum: 0 trim-ninja-timeseries: type: boolean description: If true, trims renewables.ninja time series to the year in question diff --git a/docs/about/references.md b/docs/about/references.md index f8240171..fe3c38b6 100644 --- a/docs/about/references.md +++ b/docs/about/references.md @@ -60,3 +60,7 @@ Barkatullah, N. and Ahmad, A. (2017). Current status and emerging trends in f ### @BEIS:2016 Department for Business, Energy and Industrial Strategy (2016). Electricity Generation Costs 2016. [https://www.gov.uk/government/publications/beis-electricity-generation-costs-november-2016](https://www.gov.uk/government/publications/beis-electricity-generation-costs-november-2016) + +### @Mantzos:2017 + +Mantzos, L., Wiesenthal, T., Matei, N. A., Tchung-Ming, S., Rozsai, M., Russ, P., & Ramirez, A. S. (2017). JRC-IDEES: Integrated Database of the European Energy Sector: Methodological note. JRC Research Reports, Article JRC108244. https://ideas.repec.org//p/ipt/iptwpa/jrc108244.html diff --git a/docs/model/customisation.md b/docs/model/customisation.md index 35ae4c95..d4bfd631 100644 --- a/docs/model/customisation.md +++ b/docs/model/customisation.md @@ -38,6 +38,18 @@ Here, we describe each module in terms of the technologies they contain (`callio **demand_elec**: Electricity demand +??? note "demand/electrified_transport.yaml" + + === "Technologies" + + **demand_road_transport_electrified**: Electrified road transport demand + + **demand_road_transport_historic_electrified**: Removes historically electrified road transport demand to avoid double counting + + === "Overrides" + + **keep-historic-electricity-demand-from-road-transport**: Keep historically electrified road transport demand. Historically electrified road transport demand is deleted by default, as it is already considered in historic electricity demand and would thus be counted twice. Using this override together with Euro-Calliope's default electricity demand is not advised. + ??? note "storage/electricity.yaml" === "Technologies" diff --git a/docs/model/overview.md b/docs/model/overview.md index 67be486e..1a07a5cc 100644 --- a/docs/model/overview.md +++ b/docs/model/overview.md @@ -16,6 +16,8 @@ Within each resolution-specific model directory, there is a subdirectory for tec | | | └── capacityfactors-{technology}.csv <- Timeseries of capacityfactors of all renewables. | | └── demand | | | └── electricity.csv <- Timeseries of electricity demand on each node. +| | | └── electrified-road-transport.csv <- Timeseries of road transport demand electrified on each node. +| | | └── road-transport-historic-electrification.csv <- Timeseries of historically electrified road transport demand on each node. | ├── techs <- All technology definition YAML files. | | |── {technology-class} <- Calliope base technology classes (one of `supply`, `demand`, `storage`, `transmission`). | | | └── {technology-group}.yaml <- Definition of a technology (or group of technologies) relevant to the base technology, and the allocation of that technology to nodes in the model. diff --git a/envs/default.yaml b/envs/default.yaml index c1f26db7..bc9b1a39 100644 --- a/envs/default.yaml +++ b/envs/default.yaml @@ -2,7 +2,7 @@ name: default channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 - ipython=7.21.0 - numpy=1.20.2 - pandas=1.2.3 @@ -11,5 +11,8 @@ dependencies: - pycountry=18.12.8 - jinja2=2.11.3 - pip=21.0.1 + - xarray=0.17.0 + - netCDF4=1.5.6 - pip: - -e ./lib + - styleframe==4.2 diff --git a/envs/geo.yaml b/envs/geo.yaml index af3cc9d8..16549fec 100644 --- a/envs/geo.yaml +++ b/envs/geo.yaml @@ -2,7 +2,7 @@ name: geo channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 - numpy=1.20.2 - scipy=1.6.2 - pandas=1.2.3 @@ -16,7 +16,7 @@ dependencies: - netcdf4=1.5.6 - xarray=0.17.0 - jinja2=2.11.3 - - networkx=2.2 + - networkx=2.5 - pycountry=18.12.8 - pip=21.0.1 - pip: diff --git a/envs/hydro.yaml b/envs/hydro.yaml index 1a6e8810..65f6c79a 100644 --- a/envs/hydro.yaml +++ b/envs/hydro.yaml @@ -2,7 +2,7 @@ name: hydro channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 - numpy=1.20.2 - pandas=1.2.3 - gdal=3.3.1 diff --git a/envs/vis.yaml b/envs/vis.yaml index dc02fbb7..5cf8bc77 100644 --- a/envs/vis.yaml +++ b/envs/vis.yaml @@ -2,7 +2,7 @@ name: vis channels: - conda-forge dependencies: - - python=3.8 + - python=3.9 - ipython=7.21.0 - numpy=1.20.2 - pandas=1.2.3 diff --git a/lib/eurocalliopelib/utils.py b/lib/eurocalliopelib/utils.py index 1c4133ef..d47ad853 100644 --- a/lib/eurocalliopelib/utils.py +++ b/lib/eurocalliopelib/utils.py @@ -7,10 +7,61 @@ def eu_country_code_to_iso3(eu_country_code): The European Union uses its own country codes, which often but not always match ISO 3166. """ assert len(eu_country_code) == 2, "EU country codes are of length 2, yours is '{}'.".format(eu_country_code) - if eu_country_code.lower() == "el": - iso2 = "gr" - elif eu_country_code.lower() == "uk": - iso2 = "gb" - else: - iso2 = eu_country_code - return pycountry.countries.lookup(iso2).alpha_3 + + return convert_country_code(eu_country_code, output="alpha3") + + +def convert_country_code(input_country, output="alpha3"): + """ + Converts input country code or name into either a 2- or 3-letter code. + + ISO alpha2: alpha2 + ISO alpha2 with Eurostat codes: alpha2_eu + ISO alpha3: alpha3 + + """ + + if input_country.lower() == "el": + input_country = "gr" + elif input_country.lower() == "uk": + input_country = "gb" + elif ( + input_country.lower() == "bh" + ): # this is a weird country code used in the biofuels dataset + input_country = "ba" + + if output == "alpha2": + return pycountry.countries.lookup(input_country).alpha_2 + + if output == "alpha2_eu": + result = pycountry.countries.lookup(input_country).alpha_2 + if result == "GB": + return "UK" + elif result == "GR": + return "EL" + else: + return result + + if output == "alpha3": + return pycountry.countries.lookup(input_country).alpha_3 + + +# conversion utils +def ktoe_to_twh(array): + """Convert KTOE to TWH""" + return array * 1.163e-2 + + +def pj_to_twh(array): + """Convert PJ to TWh""" + return array / 3.6 + + +def tj_to_twh(array): + """Convert TJ to TWh""" + return pj_to_twh(array) / 1000 + + +def gwh_to_tj(array): + """Convert GWh to TJ""" + return array * 3.6 diff --git a/rules/data.smk b/rules/data.smk new file mode 100644 index 00000000..4df5859a --- /dev/null +++ b/rules/data.smk @@ -0,0 +1,87 @@ +"Rules downloading and preprocessing data that is used in different places of the workflow" +# TODO consider merging shape.smk into this + +"Rules regarding CH Data:" + +rule download_ch_energy_balances: + message: "Get {wildcards.dataset} from Swiss statistics" + params: + url = lambda wildcards: config["data-sources"][f"swiss-{wildcards.dataset}"] + output: protected("data/automatic/ch-{dataset}.xlsx") + wildcard_constraints: + dataset = "((energy-balance)|(industry-energy-balance))" + localrule: True + shell: "curl -sLo {output} {params.url}" + + +"Rules regarding Eurostat Data:" + + +rule download_eurostat_annual_energy_balances: + message: "Download Eurostat Annual Energy Balances from euro-calliope datasets" + params: + url = config["data-sources"]["eurostat-energy-balance"] + output: protected("data/automatic/eurostat-energy-balance.tsv.gz") + localrule: True + shell: "curl -sLo {output} {params.url}" + + +rule annual_energy_balances: + message: "Get annual energy balances from Eurostat" + input: + energy_balance = "data/automatic/eurostat-energy-balance.tsv.gz", + ch_energy_balance = "data/automatic/ch-energy-balance.xlsx", + ch_industry_energy_balance = "data/automatic/ch-industry-energy-balance.xlsx", + cat_names = "config/energy-balances/energy-balance-category-names.csv", + carrier_names = "config/energy-balances/energy-balance-carrier-names.csv" + output: "build/data/annual-energy-balances.csv" + params: + first_year = 2000 + conda: "../envs/default.yaml" + script: "../scripts/data/annual_energy_balance.py" + + +"Rules regarding JRC-IDEES Data:" + + +rule download_jrc_idees_zipped: + message: "Download JRC IDEES zip file for {wildcards.country_code}" + params: url = config["data-sources"]["jrc-idees"] + output: protected("data/automatic/jrc-idees/{country_code}.zip") + conda: "../envs/shell.yaml" + localrule: True + shell: "curl -sLo {output} '{params.url}'" + + +rule jrc_idees_unzipped: + message: "Unzip all JRC-IDEES {wildcards.sector} sector country data" + input: + "data/automatic/jrc-idees/{country_code}.zip" + params: sector_title_case = lambda wildcards: wildcards.sector.title() + wildcard_constraints: + sector = "(transport)" + output: temp("build/data/jrc-idees/{sector}/unprocessed/JRC-IDEES-2015_Transport_{country_code}.xlsx") + conda: "../envs/shell.yaml" + shadow: "minimal" + localrule: True + shell: "unzip -j {input} -d build/data/jrc-idees/{wildcards.sector}/unprocessed/" + +"EU28 county codes used for downloading JRC-IDEES" +EU28 = [ + "AT", "BE", "BG", "CY", "CZ", "DE", "DK", "EE", "EL", "ES", "FI", "FR", + "HR", "HU", "IE", "IT", "LT", "LU", "LV", "MT", "NL", "PL", "PT", "RO", + "SE", "SI", "SK", "UK" +] + +rule jrc_idees_transport_processed: + message: "Process {wildcards.dataset} transport data from JRC-IDEES to be used in understanding current and future transport demand" + input: + data = expand( + "build/data/jrc-idees/transport/unprocessed/JRC-IDEES-2015_Transport_{country_code}.xlsx", + country_code=EU28 + ) + output: "build/data/jrc-idees/transport/processed-{dataset}.csv" + wildcard_constraints: + dataset = "((road-energy)|(road-distance)|(road-vehicles))" + conda: "../envs/default.yaml" + script: "../scripts/transport/jrc_idees.py" diff --git a/rules/transport.smk b/rules/transport.smk new file mode 100644 index 00000000..70556c48 --- /dev/null +++ b/rules/transport.smk @@ -0,0 +1,82 @@ +"""Rules to process transport sector data.""" + + +rule annual_transport_demand: + message: "Calculate future transport energy demand based on JRC IDEES" + input: + energy_balances = "build/data/annual-energy-balances.csv", + jrc_road_energy = "build/data/jrc-idees/transport/processed-road-energy.csv", + jrc_road_distance = "build/data/jrc-idees/transport/processed-road-distance.csv", + jrc_road_vehicles = "build/data/jrc-idees/transport/processed-road-vehicles.csv", + params: + fill_missing_values = config["parameters"]["transport"]["fill-missing-values"], + efficiency_quantile = config["parameters"]["transport"]["future-vehicle-efficiency-percentile"] + conda: "../envs/default.yaml" + output: + distance = "build/data/transport/annual-road-transport-distance-demand.csv", + vehicles = "build/data/transport/annual-road-transport-vehicles.csv", + efficiency = "build/data/transport/annual-road-transport-efficiency.csv", + road_electricity_historic = "build/data/transport/annual-road-transport-historic-electrification.csv", + script: "../scripts/transport/annual_transport_demand.py" + + +rule create_road_transport_timeseries: + message: "Create timeseries for road transport demand" + input: + data = "build/data/transport/annual-road-transport-distance-demand.csv", + params: + first_year = config["scope"]["temporal"]["first-year"], + final_year = config["scope"]["temporal"]["final-year"], + power_scaling_factor = config["scaling-factors"]["power"], + conversion_factor = lambda wildcards: config["parameters"]["transport"]["road-transport-conversion-factors"][wildcards.type], + type_name = lambda wildcards: config["parameters"]["transport"]["names"][wildcards.type], + historic = False + conda: "../envs/default.yaml" + wildcard_constraints: + type = "light-duty-vehicles|heavy-duty-vehicles|coaches-and-buses|passenger-cars|motorcycles" + output: + main = "build/data/transport/timeseries/timeseries-{type}.csv", + script: "../scripts/transport/road_transport_timeseries.py" + + +use rule create_road_transport_timeseries as create_road_transport_timeseries_historic_electrification with: + message: "Create timeseries for historic electrified road transport demand" + input: + data = "build/data/transport/annual-road-transport-historic-electrification.csv" + params: + first_year = config["scope"]["temporal"]["first-year"], + final_year = config["scope"]["temporal"]["final-year"], + power_scaling_factor = config["scaling-factors"]["power"], + conversion_factor = lambda wildcards: config["parameters"]["transport"]["road-transport-conversion-factors"][wildcards.type], + type_name = lambda wildcards: config["parameters"]["transport"]["names"][wildcards.type], + historic = True + output: + "build/data/transport/timeseries/timeseries-{type}-historic-electrification.csv" + + +rule aggregate_timeseries: # TODO consider merge with other rules, as this is tiny atm + message: "Aggregates timeseries for {wildcards.resolution} electrified road transport transport" + input: + time_series = ( + "build/data/transport/timeseries/timeseries-light-duty-vehicles.csv", + "build/data/transport/timeseries/timeseries-heavy-duty-vehicles.csv", + "build/data/transport/timeseries/timeseries-coaches-and-buses.csv", + "build/data/transport/timeseries/timeseries-passenger-cars.csv", + "build/data/transport/timeseries/timeseries-motorcycles.csv"), + params: + countries = config["scope"]["spatial"]["countries"] + conda: "../envs/default.yaml" + output: + "build/models/{resolution}/timeseries/demand/electrified-road-transport.csv", + script: "../scripts/transport/aggregate_timeseries.py" + + +use rule aggregate_timeseries as aggregate_timeseries_historic_electrified with: + message: "Aggregates timeseries for {wildcards.resolution} historically electrified road transport" + input: + time_series = ( + "build/data/transport/timeseries/timeseries-light-duty-vehicles-historic-electrification.csv", + "build/data/transport/timeseries/timeseries-coaches-and-buses-historic-electrification.csv", + "build/data/transport/timeseries/timeseries-passenger-cars-historic-electrification.csv"), + output: + "build/models/{resolution}/timeseries/demand/road-transport-historic-electrification.csv" diff --git a/scripts/__builtins__.py b/scripts/__builtins__.py new file mode 100644 index 00000000..5cdc9bbd --- /dev/null +++ b/scripts/__builtins__.py @@ -0,0 +1,2 @@ +class snakemake: # avoids linting errors with the injected snakemake object + pass diff --git a/scripts/data/annual_energy_balance.py b/scripts/data/annual_energy_balance.py new file mode 100644 index 00000000..30486705 --- /dev/null +++ b/scripts/data/annual_energy_balance.py @@ -0,0 +1,325 @@ +from string import digits +from enum import Enum + +import pandas as pd + +from eurocalliopelib import utils + +idx = pd.IndexSlice + + +class CAT_CODE(Enum): + FINAL_CONSUMPTION_HOUSEHOLD_CATEGORY = "FC_OTH_HH_E" + FINAL_CONSUMPTION_INDUSTRY_CATEGORY = "FC_IND_E" + FINAL_CONSUMPTION_OTHER_SECTORS_COMMERCIAL_PUBLIC_SERVICES = "FC_OTH_CP_E" + + +def generate_annual_energy_balance_nc( + path_to_energy_balance: str, path_to_cat_names: str, path_to_carrier_names: str, path_to_ch_excel: str, + path_to_ch_industry_excel: str, path_to_result: str, first_year: int +) -> None: + """ + Open a TSV file and reprocess it into a xarray dataset, including long names for + Eurostat codes. + Switzerland is not included in Eurostat, so we splice in data from their govt. + statistics. + """ + # Names for each consumption category/sub-category and carriers have been prepared by hand + cat_names = pd.read_csv(path_to_cat_names, header=0, index_col=0) + carrier_names = pd.read_csv(path_to_carrier_names, header=0, index_col=0) + + df = pd.read_csv( + path_to_energy_balance, + delimiter='\t', + index_col=0, + na_values=[":", ": ", ": z"] + ) + df.index = ( + df + .index + .str + .split(',', expand=True) + .rename(['cat_code', 'carrier_code', 'unit', 'country']) # comes as 'nrg_bal,siec,unit,geo\\time' + ) + not_countries = [c for c in df.reset_index().country.unique() if len(c) > 2] + ["XK"] + df = ( + df + .drop(axis=0, level="country", labels=not_countries) + .reset_index(level="country") + .assign(country=lambda df: df.country.map(utils.convert_country_code)) + .set_index("country", append=True) + ) + df.columns = df.columns.astype(int).rename('year') + df = df.loc[ + idx[cat_names.index, carrier_names.index, 'TJ'], : + ].dropna(how='all') + df = df.sort_index(axis=1).loc[:, first_year:] + + tdf = df.stack() + + # Add CH energy use (only covers a subset of sectors and carriers, but should be enough) + ch_energy_use_tdf = add_ch_energy_balance( + path_to_ch_excel, path_to_ch_industry_excel, index_levels=tdf.index.names + ) + tdf = pd.concat([tdf, ch_energy_use_tdf]).sort_index(axis=0) + + # TODO treat missing values if necessary + + tdf.rename("value").to_csv(path_to_result) + + +def add_ch_energy_balance(path_to_ch_excel, path_to_ch_industry_excel, index_levels): + household_sheet = 'T17a' + industry_sheet = 'T17b' + other_sectors_sheet = 'T17c' + + ch_hh_energy_use = get_ch_energy_balance_sheet( + path_to_ch_excel, household_sheet, skipfooter=9, cat_code=CAT_CODE.FINAL_CONSUMPTION_HOUSEHOLD_CATEGORY + ) + ch_ind_energy_use = get_ch_energy_balance_sheet( + path_to_ch_excel, industry_sheet, skipfooter=12, cat_code=CAT_CODE.FINAL_CONSUMPTION_INDUSTRY_CATEGORY + ) + ch_ser_energy_use = get_ch_energy_balance_sheet( + path_to_ch_excel, other_sectors_sheet, skipfooter=12, + cat_code=CAT_CODE.FINAL_CONSUMPTION_OTHER_SECTORS_COMMERCIAL_PUBLIC_SERVICES + ) + + ch_waste_energy_use = get_ch_waste_consumption(path_to_ch_excel) + ch_industry_subsector_energy_use = get_ch_industry_energy_balance(path_to_ch_industry_excel) + ch_transport_energy_use = get_ch_transport_energy_balance(path_to_ch_excel) + + ch_energy_use_tdf = pd.concat([ + df + .reset_index('year') + .assign(country='CHE', unit='TJ') + .set_index(['year', 'country', 'unit'], append=True) + .squeeze() + .reorder_levels(index_levels) + for df in [ + ch_hh_energy_use, ch_ind_energy_use, ch_ser_energy_use, ch_waste_energy_use, + ch_industry_subsector_energy_use, ch_transport_energy_use + ] + ]) + + return ch_energy_use_tdf + + +def get_ch_energy_balance_sheet(path_to_excel, sheet, skipfooter, cat_code): + ch_energy_carriers = { + 'Erdölprodukte': 'O4000XBIO', + 'Elektrizität': 'E7000', + 'Gas': 'G3000', + 'Kohle': 'C0000X0350-0370', + 'Holzenergie': 'R5110-5150_W6000RI', + 'Fernwärme': 'H8000', + 'Industrieabfälle': 'W6100_6220', + 'Übrige erneuerbare Energien': 'RA000', + 'Total\n= %': 'TOTAL', + } + # Footnote labels lead to some strings randomly ending in numbers; we remove them here + remove_digits = str.maketrans('', '', digits) + df = ( + pd + .read_excel( + path_to_excel, + skiprows=6, + skipfooter=skipfooter, + index_col=0, + sheet_name=sheet, + dtype="float", + na_values=["-"], + header=[0, 1, 2, 3, 4] + ) + .xs('TJ', level=-1, axis=1) # Ignore columns giving % use + .drop(columns=["Erdölprodukte", "Erdölprodukte1"], level=0, errors="ignore") # ignore the column giving subset of oil use which is light oil + .rename_axis(index='year') + ) + df.columns = ( + df + .columns + .get_level_values(0) + .str + .translate(remove_digits) + .map(ch_energy_carriers) + .rename('carrier_code') + ) + + return ( + df + .assign(cat_code=cat_code) + .set_index("cat_code", append=True) + .stack() + ) + + +def get_ch_waste_consumption(path_to_excel): + """ + In a different sheet in the CH GEST dataset, get data on the consumed quantity of + waste burned in WtE plants, ignoring the small (~2-3%) quantity of fossil fuels + also consumed in WtE plants to kickstart the process. + ASSUME: Small quantity (~2-3%) of fossil fuels consumed in Swiss WtE plants can be ignored. + """ + category_code = "TI_EHG_E" + carrier_code = "W6100_6220" + sheet_name = 'T27' + + waste_stream_gwh = pd.read_excel( + path_to_excel, sheet_name=sheet_name, + skiprows=5, index_col=0, header=[0, 1], skipfooter=8 + )[("Consommation d'énergie (GWh)", "Ordures")] + waste_stream_tj = waste_stream_gwh.apply(utils.gwh_to_tj) + waste_stream_tdf = ( + waste_stream_tj + .to_frame(carrier_code) # carrier code + .rename_axis(index="year", columns="carrier_code") + .assign(cat_code=category_code) # cat code + .set_index("cat_code", append=True) + .stack() + ) + return waste_stream_tdf + + +def get_ch_transport_energy_balance(path_to_excel): + carriers = { + 'Elektrizität': 'E7000', + 'Gas übriger Vekehr': 'G3000', + 'Übrige erneuerbare Energien': 'R5220B', + 'davon Benzin': 'O4652XR5210B', + 'davon Diesel': 'O4671XR5220B', + 'davon Flugtreibstoffe': 'O4000XBIO' + } + categories = { + 'O4652XR5210B': 'FC_TRA_ROAD_E', + 'O4671XR5220B': 'FC_TRA_ROAD_E', + 'R5220B': 'FC_TRA_ROAD_E', + 'G3000': 'FC_TRA_ROAD_E', + 'E7000': 'FC_TRA_RAIL_E', + 'O4000XBIO': 'INTAVI' + } + df = ( + pd + .read_excel( + path_to_excel, + skiprows=6, + skipfooter=12, + index_col=0, + sheet_name='T17e', + dtype="float", + na_values=["-"], + header=[0, 1, 2, 3, 4] + ) + .xs('TJ', level=-1, axis=1) + ) + # Footnote labels lead to some strings randomly ending in numbers; we remove them here + remove_digits = str.maketrans('', '', digits) + # carrier names span across two column levels, which we merge with fillna + carrier_name_func = ( + lambda index: + df.columns.to_frame().iloc[:, index].str.translate(remove_digits).map(carriers) + ) + df.columns = carrier_name_func(0).fillna(carrier_name_func(1)).values + + return ( + df + .groupby(axis=1, level=0).sum() + .rename_axis(index='year', columns='carrier_code') + .T + .assign(cat_code=lambda df: df.index.map(categories)) + .set_index('cat_code', append=True) + .stack() + ) + + +def get_ch_industry_energy_balance(path_to_excel): + ch_subsectors = { + "1 Nahrg.": 'FC_IND_FBT_E', # 'Food, beverages & tobacco', + "2 Textil": 'FC_IND_TL_E', # 'Textile & leather', + "3 Papier": 'FC_IND_PPP_E', # 'Paper, pulp & printing', + "4 Chemie": 'FC_IND_CPC_E', # 'Chemical & petrochemical', + "5 Zement": 'FC_IND_NMM_E', # 'Non-metallic minerals', + "6 andere": 'FC_IND_NMM_E', # 'Non-metallic minerals', + "7 Metall": 'FC_IND_IS_E', # 'Iron & steel', + "8 NE": 'FC_IND_NFM_E', # 'Non-ferrous metals', + "9 Metall": 'FC_IND_MAC_E', # 'Machinery', + "10 Masch": 'FC_IND_MAC_E', # 'Machinery', + "11 and.": 'FC_IND_NSP_E', + # 'Not elsewhere specified (industry)', 'Wood & wood products, Mining & quarrying, Transport equipment + "12 Bau": 'FC_IND_CON_E' # 'Construction' + } + + ch_carriers = { # first row in which carriers are defined in the file + 25: 'E7000', # 'electricity', + 52: 'O4000XBIO', # 'oil', + 79: 'G3000', # 'gas', + 105: 'C0000X0350-0370', # 'solid_fuel', + 128: 'W6100_6220', # 'waste', + 151: 'O4000XBIO', # 'oil', + 191: 'H8000', # 'heat', # purchased + 228: 'R5110-5150_W6000RI', # 'biofuel' + } + + column_names = ( + pd + .read_excel( + path_to_excel, + sheet_name='Überblick_tot', + skiprows=5, + usecols="A,E:P", + nrows=0, + header=0 + ) + .rename(columns=ch_subsectors) + .iloc[:, 1:] # ignore index column + ) + column_names = ["year"] + list(column_names) + + return ( + pd + .concat( + [ + read_industry_subsector(path_to_excel, first_row, column_names, carrier_name) + for first_row, carrier_name in ch_carriers.items() + ], + axis=0 + ) + .groupby(axis=0, level=[0, 1]) # group carriers + .sum() + .stack() + .rename("value") + ) + + +def read_industry_subsector(path: str, first_row: int, column_names: list[str], carrier_code: str) -> pd.DataFrame: + return ( + pd + .read_excel( + path, + sheet_name='Überblick_tot', + skiprows=first_row - 1, + usecols="A,E:P", + nrows=10, + header=None, + names=column_names, + index_col=0 + ) + .rename_axis(columns="cat_code") + .rename(index=lambda year: str(year)[:4]) + .rename(columns=lambda sector: sector.split(".")[0]) # pandas adds dots and numbers to duplicates, remove + .assign(carrier_code=carrier_code) + .reset_index() + .set_index(["carrier_code", "year"]) + .groupby(axis=1, level=0) # group sectors + .sum() + ) + + +if __name__ == "__main__": + generate_annual_energy_balance_nc( + path_to_energy_balance=snakemake.input.energy_balance, + path_to_ch_excel=snakemake.input.ch_energy_balance, + path_to_ch_industry_excel=snakemake.input.ch_industry_energy_balance, + path_to_cat_names=snakemake.input.cat_names, + path_to_carrier_names=snakemake.input.carrier_names, + first_year=snakemake.params.first_year, + path_to_result=snakemake.output[0] + ) diff --git a/scripts/transport/aggregate_timeseries.py b/scripts/transport/aggregate_timeseries.py new file mode 100644 index 00000000..42ec6f7c --- /dev/null +++ b/scripts/transport/aggregate_timeseries.py @@ -0,0 +1,41 @@ +import pandas as pd +import pycountry + + +def aggregate_timeseries(resolution: str, paths_to_input: list[str], country_codes: list[str], + path_to_output: str) -> None: + if resolution == "continental": + ts = aggregate_continental_timeseries(paths_to_input, country_codes) + elif resolution == "national": + ts = aggregate_national_timeseries(paths_to_input, country_codes) + elif resolution == "regional": + ts = aggregate_regional_timeseries(paths_to_input, country_codes) + + ts.to_csv(path_to_output) + + +def aggregate_continental_timeseries(paths_to_input: list[str], country_codes: list[str]) -> pd.DataFrame: + ts = aggregate_national_timeseries(paths_to_input, country_codes) + return ts.sum(axis=1).rename("EUR") + + +def aggregate_national_timeseries(paths_to_input: list[str], country_codes: list[str]) -> pd.DataFrame: + all_ts = [ + pd.read_csv(path, index_col='utc-timestamp', parse_dates=True) + for path in paths_to_input + ] + return sum(all_ts).loc[:, country_codes] + + +def aggregate_regional_timeseries(paths_to_input: list[str], country_codes: list[str]) -> pd.DataFrame: + # TODO implement + raise NotImplementedError("Regional road transport (historic) has not yet been implemented!") + + +if __name__ == "__main__": + aggregate_timeseries( + resolution=snakemake.wildcards.resolution, + paths_to_input=snakemake.input.time_series, + country_codes=[pycountry.countries.lookup(c).alpha_3 for c in snakemake.params.countries], + path_to_output=snakemake.output[0] + ) diff --git a/scripts/transport/annual_transport_demand.py b/scripts/transport/annual_transport_demand.py new file mode 100644 index 00000000..a2bb1a3d --- /dev/null +++ b/scripts/transport/annual_transport_demand.py @@ -0,0 +1,245 @@ +from enum import Enum + +import pandas as pd + +from eurocalliopelib import utils + +CARRIERS = { + 'O4652XR5210B': 'petrol', + 'R5210B': 'biofuels', + 'O4671XR5220B': 'diesel', + 'R5220B': 'biofuels', + 'O4630': 'lpg', + 'G3000': 'natural_gas', + 'E7000': 'electricity' +} + +idx = pd.IndexSlice + +YEAR_RANGE = slice(2000, 2018) + + +class FinalConsumption(str, Enum): + ROAD_TRANSPORT = "FC_TRA_ROAD_E" + OTHER_SECTORS = "FC_OTH_NSP_E" + AGRICULTURE_AND_FORESTRY = "FC_OTH_AF_E" + + +class Carrier(str, Enum): + AVIATION_GASOLINE = "O4651" + GASOLINE_JET_FUEL = "O4653" + KEROSENE_JET_FUEL = "O4661XR5230B" + OTHER_KEROSENE = "O4669" + OIL_AND_PETROLEUM = "O4000XBIO" + + +AVIATION_CARRIERS = [ + Carrier.AVIATION_GASOLINE, + Carrier.GASOLINE_JET_FUEL, + Carrier.KEROSENE_JET_FUEL, + Carrier.OTHER_KEROSENE +] + + +def get_all_distance_efficiency(energy_balance: pd.Series, + other_transport_road: pd.Series, road_energy: pd.Series, road_distance: pd.Series, + fill_missing_values: dict[str, str] + ) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + # Add transport energy demand from agriculture and 'not elsewhere specified' (military) (OTHER_TRANSPORT_ROAD) + transport_energy_balance = ( + energy_balance + .xs(FinalConsumption.ROAD_TRANSPORT) + .unstack('carrier_code') + .groupby(CARRIERS, axis=1).sum(min_count=1) + .rename_axis(columns='carrier') + .droplevel('unit') + .stack() + .add(other_transport_road, fill_value=0) + .apply(utils.tj_to_twh) + .rename_axis(index=['country_code', 'year', 'carrier']) + .unstack("year") + .loc[:, YEAR_RANGE] + .interpolate(axis=1, limit_direction="both") + .stack() + ) + + # contribution of each transport mode to carrier consumption from JRC_IDEES + # 2016-2018 from 2015 data; non-JRC countries, based on neighbour data + carrier_contribution = fill_missing_countries_and_years( + road_energy.div(road_energy.groupby(level=['carrier', 'country_code', 'year']).sum()), + fill_missing_values + ) + + # Energy consumption per transport mode by mapping transport mode + # carrier contributions to total carrier consumption + transport_energy_per_mode = carrier_contribution.mul(transport_energy_balance).dropna() + + # Distance per unit energy consumed per transport mode according to JRC IDEES + # 2016-2018 from 2015 data; non-JRC countries, based on neighbour data + transport_efficiency = fill_missing_countries_and_years( + road_distance + .where(road_distance > 0) + .div( + road_energy + .where(road_energy > 0) + .groupby(level=['country_code', 'vehicle_subtype', 'section', 'vehicle_type', 'year']) + .sum() + ), + fill_missing_values + ) + + # Distance travelled per transport mode, including years 2016-2018, + # based on JRC IDEES transport efficiency (2015 data for 2016-2018) + transport_distance_all_years = ( + transport_energy_per_mode + .groupby(level=['country_code', 'vehicle_subtype', 'section', 'vehicle_type', 'year']) + .sum() + .mul(transport_efficiency) + ) + + # Use the distance traveled based on Eurostat to fill in blanks in JRC data + aligned_dfs = ( + transport_distance_all_years + .reorder_levels(road_distance.index.names) + .align(road_distance) + ) + total_transport_distance = ( + aligned_dfs[1] + .fillna(aligned_dfs[0]) + .groupby(level=['vehicle_type', 'vehicle_subtype', 'country_code', 'year']) + .sum() + ) + + return total_transport_distance, transport_efficiency, transport_energy_per_mode + + +def get_all_vehicles(total_road_distance: pd.DataFrame, road_vehicles: pd.Series, + fill_missing_values: dict[str, str]) -> tuple[pd.DataFrame, pd.DataFrame]: + total_vehicle_distance = fill_missing_countries_and_years( + road_vehicles.div(road_vehicles), + fill_missing_values + ) + total_road_vehicles = total_road_distance.mul( + total_vehicle_distance + .mean(level=['vehicle_type', 'vehicle_subtype', 'country_code', 'year']) + ) + total_road_distance = total_road_distance.sum( + level=['vehicle_type', 'country_code', 'year'] + ) + total_road_vehicles = total_road_vehicles.sum( + level=['vehicle_type', 'country_code', 'year'] + ) + + return total_road_vehicles, total_road_distance + + +def fill_missing_countries_and_years(jrc_data: pd.DataFrame, fill_missing_values: dict[str, str]) -> pd.DataFrame: + jrc_data = jrc_data.unstack('country_code') + try: + jrc_data = jrc_data.loc[:, "value"] + except KeyError: + pass # it's fine. Just checking there is no MultiIndex in the columns + for country, neighbors in fill_missing_values.items(): + jrc_data = jrc_data.assign(**{country: jrc_data[neighbors].mean(axis=1)}) + + jrc_data = jrc_data.stack().unstack('year') + jrc_data = jrc_data.assign( + **{str(year): jrc_data[2015] for year in range(2016, 2019)} + ) + jrc_data.columns = jrc_data.columns.astype(int) + return jrc_data.stack() + + +def road_efficiency_cleanup(road_efficiency: pd.DataFrame, efficiency_quantile: float) -> pd.DataFrame: + return ( + (1 / road_efficiency.xs(2015, level='year')) + .unstack(['vehicle_type', 'vehicle_subtype']) + .quantile(efficiency_quantile) + .unstack(0) + .groupby({ + 'Diesel oil engine': 'diesel', + 'Battery electric vehicles': 'electricity', + 'Domestic': 'diesel', + 'International': 'diesel', + 'Gasoline engine': 'petrol', + 'Plug-in hybrid electric': 'petrol' + }).mean() + .stack() + ) + + +def get_historic_road_electricity_consumption(road_historic_consumption: pd.DataFrame) -> pd.DataFrame: + return ( + road_historic_consumption + .groupby(level=['carrier', 'vehicle_type', 'country_code', 'year']) + .sum() + .xs('electricity') + ) + + +if __name__ == "__main__": + energy_balances = pd.read_csv( + snakemake.input.energy_balances, + index_col=["cat_code", "carrier_code", "unit", "country", "year"] + ).squeeze() + road_energy = pd.read_csv( + snakemake.input.jrc_road_energy, + index_col=["section", "vehicle_type", "vehicle_subtype", "carrier", "country_code", "year"] + ).squeeze() + road_distance = pd.read_csv( + snakemake.input.jrc_road_distance, + index_col=["section", "vehicle_type", "vehicle_subtype", "country_code", "year"] + ).squeeze() + road_vehicles = pd.read_csv( + snakemake.input.jrc_road_vehicles, + index_col=["section", "vehicle_type", "vehicle_subtype", "country_code", "year"] + ) + # Used to add transport energy demand from agriculture and 'not elsewhere specified' (military) + # ASSUME: agriculture oil use goes to 'road' transport demand; + # 'not elsewhere specified' oil use goes predominantly to 'road' transport, except kerosene which goes to aviation + other_transportation_aviation = ( # all kerosene from the military destined for aviation + energy_balances + .loc[idx[FinalConsumption.OTHER_SECTORS, AVIATION_CARRIERS, :, :, :]] + .groupby(level=["country", "year"]) + .sum() + ) + other_transport_road = ( # i.e. all oil that isn't destined for aviation + energy_balances + .loc[idx[[FinalConsumption.AGRICULTURE_AND_FORESTRY, FinalConsumption.OTHER_SECTORS], Carrier.OIL_AND_PETROLEUM, :, :, :]] + .groupby(level=["country", "year"]) + .sum() + .sub(other_transportation_aviation, fill_value=0) # remove fuel use assumed for aviation + .to_frame('diesel') + .rename_axis(columns='carrier') + .stack() + ) + + fill_missing_values = snakemake.params.fill_missing_values + + # Calculate total road distance, road efficiency and historically electrified road consumption + total_road_distance, road_efficiency, road_historically_electrified_consumption = get_all_distance_efficiency( + energy_balance=energy_balances, + other_transport_road=other_transport_road, + road_energy=road_energy, + road_distance=road_distance, + fill_missing_values=fill_missing_values + ) + + # Calculate total road vehicles and aggregate total road distance + total_road_vehicles, total_road_distance = get_all_vehicles( + total_road_distance, + road_vehicles, + fill_missing_values + ) + + # Some cleanup that's specific to road data for road efficiency + road_efficiency = road_efficiency_cleanup(road_efficiency, snakemake.params.efficiency_quantile) + + # Extract historical electricity consumption + road_electricity_historic = get_historic_road_electricity_consumption(road_historically_electrified_consumption) + + # Create CSV Files for calculated data + total_road_distance.rename("value").to_csv(snakemake.output.distance) + total_road_vehicles.rename("value").to_csv(snakemake.output.vehicles) + road_efficiency.rename("value").to_csv(snakemake.output.efficiency) + road_electricity_historic.rename("value").to_csv(snakemake.output.road_electricity_historic) diff --git a/scripts/transport/jrc_idees.py b/scripts/transport/jrc_idees.py new file mode 100644 index 00000000..ddcb76ae --- /dev/null +++ b/scripts/transport/jrc_idees.py @@ -0,0 +1,180 @@ +from pathlib import Path + +import numpy as np +import pandas as pd +from styleframe import StyleFrame + +from eurocalliopelib import utils + +idx = pd.IndexSlice + +DATASET_PARAMS = { + "road-energy": { + "sheet_name": "TrRoad_ene", + "idx_start_str": "Total energy consumption", + "idx_end_str": "Indicators", + "unit": "ktoe" + }, + "road-distance": { + "sheet_name": "TrRoad_act", + "idx_start_str": "Vehicle-km driven", + "idx_end_str": "Stock of vehicles", + "unit": "mio_km" + }, + "road-vehicles": { + "sheet_name": "TrRoad_act", + "idx_start_str": "Stock of vehicles - in use", + "idx_end_str": "New vehicle-registrations", + "unit": "N. vehicles" + } +} + +ROAD_CARRIERS = { + 'Gasoline engine': 'petrol', + 'Diesel oil engine': 'diesel', + 'Natural gas engine': 'natural_gas', + 'LPG engine': 'lpg', + 'Battery electric vehicles': 'electricity', + 'Plug-in hybrid electric': 'petrol' +} + + +def process_jrc_transport_data(paths_to_data: list[str], dataset: object, out_path: str): + paths_to_data = [Path(p) for p in paths_to_data] + processed_data = pd.concat([ + read_transport_excel(path, **DATASET_PARAMS[dataset]) + for path in paths_to_data + ]) + if DATASET_PARAMS[dataset]["unit"] == "ktoe": + processed_data = processed_data.apply(utils.ktoe_to_twh) + processed_data = ( + processed_data + .reset_index(level="country_code") + .assign(country_code=lambda df: df.country_code.map(utils.convert_country_code)) + .set_index("country_code", append=True) + .stack('year') + .rename('value') + .to_csv(out_path) + ) + + processed_data + + +def read_transport_excel(path: Path, + sheet_name: str, + idx_start_str: str, + idx_end_str: str, + **kwargs: object) -> pd.DataFrame: + xls = pd.ExcelFile(path) + style_df = StyleFrame.read_excel(xls, read_style=True, sheet_name=sheet_name) + df = pd.read_excel(xls, sheet_name=sheet_name) + column_names = str(style_df.data_df.columns[0]) + # We have manually identified the section of data which is of use to us, + # given by idx_start_str and idx_end_str. + idx_start = int(style_df[style_df[column_names].str.find(idx_start_str) > -1][0]) + idx_end = int(style_df[style_df[column_names].str.find(idx_end_str) > -1][0]) + df = df.assign(indent=style_df[column_names].style.indent.astype(int)).loc[idx_start:idx_end] + + total_to_check = df.iloc[0] + # The indent of the strings in the first column of data indicates their hierarchy in a multi-level index. + # Two levels of the hierarchy are identified here and ffill() is used to match all relevant rows + # to the top-level name. + df['section'] = df.where(df.indent == 1).iloc[:, 0].ffill() + df['vehicle_type'] = df.where(df.indent == 2).iloc[:, 0].ffill() + + if sheet_name == 'TrRoad_act': + df = process_road_vehicles(df, column_names) + elif sheet_name == 'TrRoad_ene': + df = process_road_energy(df, column_names) + df = ( + df + .assign(country_code=column_names.split(' - ')[0]) + .set_index('country_code', append=True) + ) + df.columns = df.columns.astype(int).rename('year') + + # After all this hardcoded cleanup, make sure numbers match up + assert np.allclose( + df.sum(), + total_to_check.loc[df.columns].astype(float) + ) + + return df + + +def process_road_vehicles(df: pd.DataFrame, column_names: str) -> pd.DataFrame: + # The indent of the strings in the first column of data indicates their hierarchy in a multi-level index. + # The vehicle subtype is identified here + df['vehicle_subtype'] = df.where(df.indent == 3).iloc[:, 0] + # 2-wheelers are powered by fuel oil in the dataset. + # All useful information is either when the index column string is indented 3 times, + # or when the vehicle type is a 2-wheeler. One of the many ways in which this dataset is a pain. + df = df.where( + (df.indent == 3) | (df.vehicle_type == 'Powered 2-wheelers') + ).dropna(how='all') + df.loc[df.vehicle_type == 'Powered 2-wheelers', 'vehicle_subtype'] = 'Gasoline engine' + return ( + df + .set_index(['section', 'vehicle_type', 'vehicle_subtype']) + .drop([column_names, 'indent'], axis=1) + ) + + +def process_road_energy(df: pd.DataFrame, column_names: str) -> pd.DataFrame: + # The indent of the strings in the first column of data indicates their hierarchy in a multi-level index. + # The vehicle subtype is identified here and ffill() is used to match all relevant rows to this subtype. + df['vehicle_subtype'] = df.where(df.indent == 3).iloc[:, 0].ffill() + # Remove bracketed information from the vehicle type and subtype + df['vehicle_type'] = df['vehicle_type'].str.split('(', expand=True)[0].str.strip() + df['vehicle_subtype'] = df['vehicle_subtype'].str.split('(', expand=True)[0].str.strip() + # Powered 2-wheelers are gasoline engine only (this is implicit when looking at the Excel sheet directly) + df.loc[df.vehicle_type == 'Powered 2-wheelers', 'vehicle_subtype'] = 'Gasoline engine' + df['carrier'] = df.where(df.indent == 4).iloc[:, 0] + df['carrier'] = df['carrier'].str.replace('of which ', '') + # Powered 2-wheelers use petrol, some of which is biofuels (we deal with the 'of which' part later) + df.loc[(df.vehicle_type == 'Powered 2-wheelers') & (df.indent == 2), 'carrier'] = 'petrol' + df.loc[(df.vehicle_type == 'Powered 2-wheelers') & (df.indent == 3), 'carrier'] = 'biofuels' + # both domestic and international freight uses diesel in the dataset + # (this is implicit when looking at the Excel sheet directly) + df.loc[(df.vehicle_subtype == 'Domestic') & (df.indent == 3), 'carrier'] = 'diesel' + df.loc[(df.vehicle_subtype == 'International') & (df.indent == 3), 'carrier'] = 'diesel' + # All other vehicle types mention the drive-train directly, so we translate that to energy carrier here + df['carrier'] = df['carrier'].fillna(df.vehicle_subtype.replace(ROAD_CARRIERS)) + + df = ( + df + .where((df.indent > 2) | (df.vehicle_type == 'Powered 2-wheelers')) + .dropna() + .set_index(['section', 'vehicle_type', 'vehicle_subtype', 'carrier']) + .drop([column_names, 'indent'], axis=1) + ) + + df = remove_of_which(df, 'diesel', 'biofuels') + df = remove_of_which(df, 'petrol', 'biofuels') + df = remove_of_which(df, 'petrol', 'electricity') + df = remove_of_which(df, 'natural_gas', 'biogas') + + return df + + +def remove_of_which(df: pd.DataFrame, main_carrier: str, of_which_carrier: str) -> pd.DataFrame: + """ + Subfuels (e.g. biodiesel) are given as 'of which ...', meaning the main fuel consumption + includes those fuels (diesel is actually diesel + biofuels). We rectify that here + """ + updated_carrier = ( + (df.xs(main_carrier, level='carrier') - df.xs(of_which_carrier, level='carrier')) + .dropna() + .assign(carrier=main_carrier) + .set_index('carrier', append=True) + ) + df.loc[updated_carrier.index] = updated_carrier + return df + + +if __name__ == "__main__": + process_jrc_transport_data( + paths_to_data=snakemake.input.data, + dataset=snakemake.wildcards.dataset, + out_path=snakemake.output[0] + ) diff --git a/scripts/transport/road_transport_timeseries.py b/scripts/transport/road_transport_timeseries.py new file mode 100644 index 00000000..4c75517d --- /dev/null +++ b/scripts/transport/road_transport_timeseries.py @@ -0,0 +1,52 @@ +import pandas as pd + + +def create_road_transport_demand_timeseries( + path_to_input: str, first_year: int, final_year: int, type_name: str, + power_scaling_factor: float, conversion_factor: float, historic: bool, path_to_output: str +) -> None: + # Read annual road transport distance into panda dataframe + df = pd.read_csv(path_to_input, index_col=[0, 1, 2]) + + # ASSUME: Every hour of the year the same amount of distance gets driven. + # Calculate the distance travelled per hour + series = df["value"] / 8760 + + # Process the road transport distance timeseries + create_timeseries(type_name, power_scaling_factor, conversion_factor, first_year, + final_year, series, path_to_output, historic=historic) + + +def create_timeseries(vehicle: str, power_scaling_factor: float, conversion_factor: float, + first_year: int, final_year: int, series: pd.Series, + output_path: str, historic: bool): + ts_index = pd.to_datetime(range(first_year, final_year + 2), format="%Y") + ts = ( + series + .xs(vehicle) + .xs(slice(first_year, final_year + 1), level=1, drop_level=False) + .unstack("country_code") + .set_index(ts_index) + .resample("H") + .ffill() + .iloc[:-1] # remove first hour in following year + .mul(conversion_factor) + .mul(power_scaling_factor) + .mul(1 if historic else -1) # historic demand is actually a supply to avoid double counting + ) + ts_index = pd.to_datetime(ts.index, format="%Y") + ts = ts.set_index(ts_index) + ts.to_csv(output_path, index_label="utc-timestamp") + + +if __name__ == "__main__": + create_road_transport_demand_timeseries( + path_to_input=snakemake.input.data, + power_scaling_factor=snakemake.params.power_scaling_factor, + first_year=snakemake.params.first_year, + final_year=snakemake.params.final_year, + type_name=snakemake.params.type_name, + conversion_factor=snakemake.params.conversion_factor, + historic=snakemake.params.historic, + path_to_output=snakemake.output[0], + ) diff --git a/templates/models/techs/demand/electrified-transport.yaml b/templates/models/techs/demand/electrified-transport.yaml new file mode 100644 index 00000000..aeffe7d0 --- /dev/null +++ b/templates/models/techs/demand/electrified-transport.yaml @@ -0,0 +1,29 @@ +techs: + demand_road_transport_electrified: + essentials: + name: 'Electrified road transport demand' + parent: demand + carrier: electricity + constraints: + resource: file=demand/electrified-road-transport.csv + + demand_road_transport_historic_electrified: + essentials: + name: 'Removes historic electrified road transport demand' + parent: supply + carrier: electricity + constraints: + resource: file=demand/road-transport-historic-electrification.csv + resource_min_use: 1 + +overrides: + keep-historic-electricity-demand-from-road-transport: + {% for id, location in locations.iterrows() %} + {{ id }}.techs.demand_road_transport_historic_electrified.exists: False + {% endfor %} + +locations: + {% for id, location in locations.iterrows() %} + {{ id }}.techs.demand_road_transport_electrified: + {{ id }}.techs.demand_road_transport_historic_electrified: + {% endfor %} diff --git a/tests/lib/test_utils.py b/tests/lib/test_utils.py index 9fb14366..1a87a2a7 100644 --- a/tests/lib/test_utils.py +++ b/tests/lib/test_utils.py @@ -1,6 +1,6 @@ import pytest -from eurocalliopelib.utils import eu_country_code_to_iso3 +from eurocalliopelib.utils import eu_country_code_to_iso3, convert_country_code @pytest.mark.parametrize('eu_country_code,iso3166', [ @@ -8,5 +8,38 @@ ("el", "GRC"), ("de", "DEU") ]) -def test_country_code_conversion(eu_country_code, iso3166): +def test_eu_country_code_conversion(eu_country_code, iso3166): assert eu_country_code_to_iso3(eu_country_code) == iso3166 + + +@pytest.mark.parametrize('input,alpha3', [ + ("uk", "GBR"), + ("el", "GRC"), + ("de", "DEU"), + ("Germany", "DEU"), + ("Greece", "GRC") +]) +def test_conversion_to_alpha3(input, alpha3): + assert convert_country_code(input, "alpha3") == alpha3 + + +@pytest.mark.parametrize('input,alpha2', [ + ("uk", "GB"), + ("el", "GR"), + ("de", "DE"), + ("Germany", "DE"), + ("Greece", "GR") +]) +def test_conversion_to_alpha2(input, alpha2): + assert convert_country_code(input, "alpha2") == alpha2 + + +@pytest.mark.parametrize('input,alpha2', [ + ("uk", "UK"), + ("el", "EL"), + ("de", "DE"), + ("Germany", "DE"), + ("Greece", "EL") +]) +def test_conversion_to_eu_alpha2(input, alpha2): + assert convert_country_code(input, "alpha2_eu") == alpha2 diff --git a/tests/model/__builtins__.py b/tests/model/__builtins__.py new file mode 100644 index 00000000..5cdc9bbd --- /dev/null +++ b/tests/model/__builtins__.py @@ -0,0 +1,2 @@ +class snakemake: # avoids linting errors with the injected snakemake object + pass diff --git a/tests/resources/test.yaml b/tests/resources/test.yaml index b89d3151..b1a3cd83 100644 --- a/tests/resources/test.yaml +++ b/tests/resources/test.yaml @@ -7,6 +7,7 @@ test-model: frozen-hydro: ["freeze-hydro-supply-capacities", "freeze-hydro-storage-capacities", "no-hydro-supply-fixed-cost", "no-hydro-storage-fixed-cost"] alternative-cost: ["dea-renewable-cost-pv-open-field", "dea-renewable-cost-wind-onshore", "dea-renewable-cost-wind-offshore", "dea-renewable-cost-pv-roof-mounted", "schroeder-hydro-cost"] shed-load: ["load-shedding"] + keep-historic-transport: ["keep-historic-electricity-demand-from-road-transport"] national: default: [] connected_neighbours: ["connect_all_neighbours"] @@ -16,6 +17,7 @@ test-model: frozen-hydro: ["freeze-hydro-supply-capacities", "freeze-hydro-storage-capacities", "no-hydro-supply-fixed-cost", "no-hydro-storage-fixed-cost"] alternative-cost: ["dea-renewable-cost-pv-open-field", "dea-renewable-cost-wind-onshore", "dea-renewable-cost-wind-offshore", "dea-renewable-cost-pv-roof-mounted", "schroeder-hydro-cost"] shed-load: ["load-shedding"] + keep-historic-transport: ["keep-historic-electricity-demand-from-road-transport"] regional: default: ["connect_all_neighbours", "run_barrier_no_crossover"] all-overrides: ["connect_all_neighbours", "directional-rooftop-pv", "exclusive-energy-to-power-ratios", diff --git a/tests/validate_schema.py b/tests/validate_schema.py index 3ef6abf3..40427670 100644 --- a/tests/validate_schema.py +++ b/tests/validate_schema.py @@ -8,7 +8,7 @@ "--config", help=( "configuration file to validate. " - "If not given, schema itself will be validated against JSON schema Draft 7" + "If not given, schema itself will be validated against JSON schema Draft 2020-12" ) ) args = parser.parse_args() @@ -21,7 +21,4 @@ config = yaml.safe_load(f) jsonschema.validate(config, schema) else: - # We set the metaschema 'additionalProperties' to False to create a 'strict' schema checker, - # which will fail on typos - jsonschema.Draft7Validator.META_SCHEMA['additionalProperties'] = False - jsonschema.Draft7Validator.check_schema(schema) + jsonschema.Draft202012Validator.check_schema(schema)