Commit 02af2d8e authored by kaghog's avatar kaghog
Browse files

Merge branch 'nfp_weekend' into nfpCovid19

Conflicts:
	synthesis/population/spatial/secondary/locations.py
parents c45ccc6c e7f26019
Pipeline #128147 failed with stage
This diff is collapsed.
# General pipeline settings
working_directory: /nas/kaghog/nfp/cache_output_m11_sat
flowchart_path: /nas/kaghog/nfp/cache_output_m11_sat/flowchart.json
working_directory: &DIR /nas/cache_output
flowchart_path: /nas/cache_output/flowchart.json
dryrun: false
# Requested stages
......@@ -8,19 +8,26 @@ run:
# - data.statpop.persons
# - data.statpop.projections.households
# - data.statpop.scaled
# - synthesis.population.matched
# - synthesis.population.matched
# - data.microcensus.households
# - data.microcensus.csv
# - population.destinations
# - synthesis.population.destinations
# - synthesis.population.spatial.primary.weekend.pry_distance_distributions
# - synthesis.population.spatial.primary.weekend.work_locations
# - synthesis.population.spatial.primary.weekend.education_locations
# - synthesis.population.spatial.secondary.distance_distributions
# - synthesis.population.spatial.secondary.locations
# - synthesis.population.spatial.primary.weekend.locations
# - matsim.facilities
# - matsim.population
# - matsim.households
# - matsim.run
- population.output
# - analysis.analysis
# - synthesis.population.spatial.primary.locations
#- synthesis.population.spatial.locations
# - matsim.scenario.population
# - matsim.scenario.households
# - matsim.scenario.facilities
- matsim.simulation.run
# - matsim.runtime.eqasim
# - population.output
# - analysis.analysis_weekend
# These are configuration options that we use in the pipeline
config:
......@@ -33,10 +40,14 @@ config:
enable_scaling: true
scaling_year: 2020
use_freight: false
use_detailed_activities: false
weekend_scenario: true
specific_weekend_scenario: saturday
use_detailed_activities: false #for a matsim run see eqasim_branch and repo below
weekend_scenario: false
specific_day_scenario: avgworkday
specific_weekend_scenario:
hafas_date: 01.10.2018
data_path: /nas/ivtmatsim/scenarios/switzerland/data
output_path: /nas/kaghog/nfp/cache_output_m11_sat
analysis_path: /nas/kaghog/nfp/cache_output_m11_sat/analysis
\ No newline at end of file
output_path: *DIR
analysis_path: /nas/cache_output/analysis
#eqasim_repository: https://github.com/kaghog/eqasim-java.git
#eqasim_branch: develop
#for eqasim version 1.3.1 only the develop branch allows for custom activities
\ No newline at end of file
# General pipeline settings
working_directory: /nas/kaghog/nfp/cache_output_m11_sun_25p
flowchart_path: /nas/kaghog/nfp/cache_output_m11_sun_25p/flowchart.json
working_directory: /nas/kaghog/nfp/cache_output_m11_mon
flowchart_path: /nas/kaghog/nfp/cache_output_m11_mon/flowchart.json
dryrun: false
# Requested stages
......@@ -8,7 +8,7 @@ run:
# - data.statpop.persons
# - data.statpop.projections.households
# - data.statpop.scaled
# - synthesis.population.matched
# - synthesis.population.matched
# - data.microcensus.households
# - data.microcensus.csv
# - population.destinations
......@@ -16,14 +16,15 @@ run:
# - synthesis.population.spatial.primary.weekend.pry_distance_distributions
# - synthesis.population.spatial.primary.weekend.work_locations
# - synthesis.population.spatial.primary.weekend.education_locations
# - synthesis.population.spatial.secondary.distance_distributions
# - synthesis.population.spatial.secondary.distance_distributions
# - synthesis.population.spatial.secondary.locations
# - synthesis.population.spatial.primary.weekend.locations
# - synthesis.population.spatial.primary.weekend.locations
# - synthesis.population.spatial.primary.locations
# - matsim.facilities
# - matsim.population
# - matsim.households
# - matsim.run
- population.output
# - population.output
# - analysis.analysis
# These are configuration options that we use in the pipeline
......@@ -33,14 +34,15 @@ config:
hot_deck_matching_runners: 2
disable_progress_bar: false
java_memory: 80G
input_downsampling: 0.25
input_downsampling: 0.1
enable_scaling: true
scaling_year: 2020
use_freight: false
use_detailed_activities: false
weekend_scenario: true
specific_weekend_scenario: sunday
use_detailed_activities: true
weekend_scenario: false
specific_day_scenario: monday #avgworkday
specific_weekend_scenario: saturday
hafas_date: 01.10.2018
data_path: /nas/ivtmatsim/scenarios/switzerland/data
output_path: /nas/kaghog/nfp/cache_output_m11_sun_25p
analysis_path: /nas/kaghog/nfp/cache_output_m11_sun_25p/analysis
\ No newline at end of file
output_path: /nas/kaghog/nfp/cache_output_m11_mon
analysis_path: /nas/kaghog/nfp/cache_output_m11_mon/analysis
\ No newline at end of file
......@@ -63,6 +63,16 @@ def execute(context):
df_mz_persons["sunday"] = False
df_mz_persons.loc[df_mz_persons["tag"] == 7, "sunday"] = True
#specify day of the week
df_mz_persons["day"] = "monday"
df_mz_persons.loc[df_mz_persons["tag"] == 2, "day"] = "tuesday"
df_mz_persons.loc[df_mz_persons["tag"] == 3, "day"] = "wednesday"
df_mz_persons.loc[df_mz_persons["tag"] == 4, "day"] = "thursday"
df_mz_persons.loc[df_mz_persons["tag"] == 5, "day"] = "friday"
df_mz_persons.loc[df_mz_persons["tag"] == 6, "day"] = "saturday"
df_mz_persons.loc[df_mz_persons["tag"] == 7, "day"] = "sunday"
# Here we extract a bit more than Kirill, but most likely it will be useful later
df_mz_persons["subscriptions_ga"] = df_mz_persons["f41610a"] == 1
......@@ -123,7 +133,7 @@ def execute(context):
"subscriptions_verbund_class",
"subscriptions_strecke_class",
"age_class", "person_weight",
"weekend", "saturday", "sunday", "date"
"weekend", "saturday", "sunday", "date", "day"
]]
# Merge in the other data sets
......
......@@ -3,44 +3,80 @@ import shutil
def configure(context):
context.stage("matsim.run")
context.stage("matsim.simulation.run")
context.stage("matsim.simulation.prepare")
context.stage("matsim.runtime.eqasim")
context.config("output_path")
context.config("output_prefix", "switzerland_")
context.config("write_jar", True)
context.config("output_id")
context.stage("contracts.contracts")
context.stage("population.output")
def execute(context):
results_path = context.stage("matsim.run")
output_path = context.config("output_path")
output_id = context.config("output_id")
if not os.path.isdir(output_path):
raise RuntimeError("Output path does not exist:", output_path)
target_path = "%s/%s" % (output_path, output_id)
context.stage("population.output")
if os.path.exists(target_path):
if os.path.isdir(target_path):
print("Cleaning target directory:", target_path)
shutil.rmtree(target_path)
else:
raise RuntimeError("Cannot clean target path:", target_path)
#context.stage("documentation.meta_output")
context.stage("contracts.contracts")
os.mkdir(target_path)
def execute(context):
config_path = "%s/%s" % (
context.path("matsim.simulation.prepare"),
context.stage("matsim.simulation.prepare")
)
for file in [
"switzerland_network.xml.gz",
"switzerland_transit_schedule.xml.gz",
"switzerland_transit_vehicles.xml.gz",
"switzerland_facilities.xml.gz",
"switzerland_households.xml.gz",
"switzerland_population.xml.gz",
"switzerland_config.xml"
for name in [
"%shouseholds.xml.gz" % context.config("output_prefix"),
"%spopulation.xml.gz" % context.config("output_prefix"),
"%sfacilities.xml.gz" % context.config("output_prefix"),
"%snetwork.xml.gz" % context.config("output_prefix"),
"%stransit_schedule.xml.gz" % context.config("output_prefix"),
"%stransit_vehicles.xml.gz" % context.config("output_prefix"),
"%sconfig.xml" % context.config("output_prefix")
]:
shutil.copyfile("%s/%s" % (results_path, file), "%s/%s" % (target_path, file))
shutil.copy(
"%s/%s" % (context.path("matsim.simulation.prepare"), name),
"%s/%s" % (context.config("output_path"), name)
)
contracts_path = context.stage("contracts.contracts")
shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % target_path)
if context.config("write_jar"):
shutil.copy(
"%s/%s" % (context.path("matsim.runtime.eqasim"), context.stage("matsim.runtime.eqasim")),
"%s/%srun.jar" % (context.config("output_path"), context.config("output_prefix"))
)
contracts_path = context.stage("contracts.contracts")
shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % (context.config("output_path"), context.config("output_prefix")))
return {}
# def execute(context):
# results_path = context.stage("matsim.run")
#
# output_path = context.config("output_path")
# output_id = context.config("output_id")
#
# if not os.path.isdir(output_path):
# raise RuntimeError("Output path does not exist:", output_path)
#
# target_path = "%s/%s" % (output_path, output_id)
#
# if os.path.exists(target_path):
# if os.path.isdir(target_path):
# print("Cleaning target directory:", target_path)
# shutil.rmtree(target_path)
# else:
# raise RuntimeError("Cannot clean target path:", target_path)
#
# os.mkdir(target_path)
#
# for file in [
# "switzerland_network.xml.gz",
# "switzerland_transit_schedule.xml.gz",
# "switzerland_transit_vehicles.xml.gz",
# "switzerland_facilities.xml.gz",
# "switzerland_households.xml.gz",
# "switzerland_population.xml.gz",
# "switzerland_config.xml"
# ]:
# shutil.copyfile("%s/%s" % (results_path, file), "%s/%s" % (target_path, file))
#
# contracts_path = context.stage("contracts.contracts")
# shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % target_path)
#
# return {}
import subprocess as sp
def configure(context):
context.stage("utils.java")
def execute(context):
java = context.stage("utils.java")
sp.check_call([
"git", "clone", "https://github.com/eqasim-org/eqasim-java.git"
], cwd=context.cache_path)
sp.check_call([
"git", "checkout", "v1.2.1"
], cwd="%s/eqasim-java" % context.cache_path)
sp.check_call([
"mvn", "-Pstandalone", "package"
], cwd="%s/eqasim-java" % context.cache_path)
jar = "%s/eqasim-java/switzerland/target/switzerland-1.2.1.jar" % context.cache_path
return jar
import os
import subprocess as sp
def configure(context):
context.stage("utils.java")
def execute(context):
java = context.stage("utils.java")
os.mkdir("%s/java_tmp" % context.cache_path)
sp.check_call([
"git", "clone", "https://github.com/matsim-org/pt2matsim.git"
], cwd=context.cache_path)
sp.check_call([
"git", "checkout", "v19.10"
], cwd="%s/pt2matsim" % context.cache_path)
sp.check_call([
"mvn", "-Djava.io.tmpdir=%s/java_tmp" % context.cache_path, "package"
], cwd="%s/pt2matsim" % context.cache_path)
jar = "%s/pt2matsim/target/pt2matsim-19.10-shaded.jar" % context.cache_path
java(jar, "org.matsim.pt2matsim.run.CreateDefaultOsmConfig", ["test_config.xml"], cwd=context.cache_path)
assert (os.path.exists("%s/test_config.xml" % context.cache_path))
assert (os.path.exists("%s/java_tmp/GeoTools" % context.cache_path))
return jar, "%s/java_tmp" % context.cache_path
......@@ -110,7 +110,7 @@ def execute(context):
person_writer.write(writer)
number_of_written_persons += 1
progress.update()
#progress.update()
except StopIteration:
pass
......
import shutil
def configure(context):
context.stage("matsim.population")
context.stage("matsim.households")
context.stage("matsim.facilities")
context.stage("matsim.network.mapped")
context.stage("matsim.java.eqasim")
context.stage("utils.java")
def execute(context):
# Some files we just copy
transit_vehicles_path = context.stage("matsim.network.mapped")["vehicles"]
shutil.copyfile(transit_vehicles_path, "%s/switzerland_transit_vehicles.xml.gz" % context.cache_path)
households_path = context.stage("matsim.households")
shutil.copyfile(households_path, "%s/switzerland_households.xml.gz" % context.cache_path)
# Some files we send through the preparation script
transit_schedule_input_path = context.stage("matsim.network.mapped")["schedule"]
transit_schedule_output_path = "%s/switzerland_transit_schedule.xml.gz" % context.cache_path
network_input_path = context.stage("matsim.network.mapped")["network"]
network_output_path = "%s/switzerland_network.xml.gz" % context.cache_path
facilities_input_path = context.stage("matsim.facilities")
facilities_output_path = "%s/switzerland_facilities.xml.gz" % context.cache_path
population_input_path = context.stage("matsim.population")
population_prepared_path = "%s/prepared_population.xml.gz" % context.cache_path
population_output_path = "%s/switzerland_population.xml.gz" % context.cache_path
config_output_path = "%s/switzerland_config.xml" % context.cache_path
# Call preparation script
java = context.stage("utils.java")
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.core.scenario.preparation.RunPreparation", [
"--input-facilities-path", facilities_input_path,
"--output-facilities-path", facilities_output_path,
"--input-population-path", population_input_path,
"--output-population-path", population_prepared_path,
"--input-network-path", network_input_path,
"--output-network-path", network_output_path,
"--threads", str(context.config("threads"))
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.core.scenario.config.RunGenerateConfig", [
"--output-path", config_output_path,
"--prefix", "switzerland_",
"--sample-size", str(context.config("input_downsampling")),
"--random-seed", str(1000),
"--threads", str(context.config("threads"))
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.switzerland.scenario.RunCalculateStopCategories", [
"--input-path", transit_schedule_input_path,
"--output-path", transit_schedule_output_path
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.core.scenario.routing.RunPopulationRouting", [
"--config-path", config_output_path,
"--output-path", population_output_path,
"--threads", str(context.config("threads")),
"--config:plans.inputPlansFile", population_prepared_path
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.core.scenario.validation.RunScenarioValidator", [
"--config-path", config_output_path
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.switzerland.scenario.RunAdaptConfig", [
"--input-path", config_output_path,
"--output-path", config_output_path
], cwd=context.cache_path)
java(
context.stage("matsim.java.eqasim"),
"org.eqasim.switzerland.RunSimulation", [
"--config-path", config_output_path,
"--config:controler.lastIteration", str(1),
"--config:controler.writeEventsInterval", str(1),
"--config:controler.writePlansInterval", str(1),
], cwd=context.cache_path)
return context.cache_path
import subprocess as sp
import os, os.path, shutil
import matsim.runtime.git as git
import matsim.runtime.java as java
import matsim.runtime.maven as maven
def configure(context):
context.stage("matsim.runtime.git")
context.stage("matsim.runtime.java")
context.stage("matsim.runtime.maven")
context.config("eqasim_version", "1.3.1")
context.config("eqasim_branch", "v1.3.1")
context.config("eqasim_repository", "https://github.com/eqasim-org/eqasim-java.git")
context.config("eqasim_path", "")
def run(context, command, arguments):
version = context.config("eqasim_version")
# Make sure there is a dependency
context.stage("matsim.runtime.eqasim")
jar_path = "%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (
context.path("matsim.runtime.eqasim"), version
)
java.run(context, command, arguments, jar_path)
def execute(context):
version = context.config("eqasim_version")
print("THIS IS A CHECK!!!")
print(context.config("eqasim_repository"))
# Normal case: we clone eqasim
if context.config("eqasim_path") == "":
# Clone repository and checkout version
git.run(context, [
"clone", context.config("eqasim_repository"),
"--branch", context.config("eqasim_branch"),
"--single-branch", "eqasim-java",
"--depth", "1"
])
# Build eqasim
maven.run(context, ["-Pstandalone", "--projects", "switzerland", "--also-make", "package"], cwd = "%s/eqasim-java" % context.path())
jar_path = "%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (context.path(), version)
# Special case: We provide the jar directly. This is mainly used for
# creating input to unit tests of the eqasim-java package.
else:
os.makedirs("%s/eqasim-java/switzerland/target" % context.path())
shutil.copy(context.config("eqasim_path"),
"%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (context.path(), version))
return "eqasim-java/switzerland/target/switzerland-%s.jar" % version
def validate(context):
path = context.config("eqasim_path")
if path == "":
return True
if not os.path.exists(path):
raise RuntimeError("Cannot find eqasim at: %s" % path)
return os.path.getmtime(path)
\ No newline at end of file
import subprocess as sp
import shutil
def configure(context):
context.config("git_binary", "git")
def run(context, arguments = [], cwd = None, catch_output = False):
"""
This function calls git.
"""
# Make sure there is a dependency
context.stage("matsim.runtime.git")
if cwd is None:
cwd = context.path()
command_line = [
shutil.which(context.config("git_binary"))
] + arguments
if catch_output:
return sp.check_output(command_line, cwd = cwd).decode("utf-8").strip()
else:
return_code = sp.check_call(command_line, cwd = cwd)
if not return_code == 0:
raise RuntimeError("Git return code: %d" % return_code)
def validate(context):
if shutil.which(context.config("git_binary")) == "":
raise RuntimeError("Cannot find git binary at: %s" % context.config("git_binary"))
if not b"2." in sp.check_output([
shutil.which(context.config("git_binary")),
"--version"
], stderr = sp.STDOUT):
print("WARNING! Git of at least version 2.x.x is recommended!")
def execute(context):
pass
\ No newline at end of file
import subprocess as sp
import os, shutil
def configure(context):
context.config("java_binary", "java")
context.config("java_memory", "50G")
def run(context, entry_point, arguments = [], class_path = None, vm_arguments = [], cwd = None, memory = None, mode = "raise"):
"""
This function calls java code. There are three modes:
- return_code: Returns the return code of the Java call
- output: Returns the output of the Java call
- raise (default): Raises an exception if the return code is not zero
"""
# Make sure there is a dependency
context.stage("matsim.runtime.java")
# Prepare temp folder
temp_path = "%s/__java_temp" % context.path()
if not os.path.exists(temp_path):
os.mkdir(temp_path)
# Prepare arguments
memory = context.config("java_memory") if memory is None else memory
vm_arguments = [
"-Xmx" + memory,
"-Djava.io.tmpdir=%s" % temp_path,
"-Dmatsim.useLocalDtds=true"
] + vm_arguments
# Prepare classpath
if type(class_path) == list or type(class_path) == tuple:
class_path = ":".join(class_path)
# Prepare CWD
if cwd is None:
cwd = context.path()
# Prepare command line
command_line = [
shutil.which(context.config("java_binary")),
"-cp", class_path
] + vm_arguments + [
entry_point
] + arguments
command_line = list(map(str, command_line))
print("Executing java:", " ".join(command_line))
if mode == "raise" or mode == "return_code":
return_code = sp.check_call(command_line, cwd = cwd)
if not return_code == 0:
raise RuntimeError("Java return code: %d" % return_code)
return return_code
elif mode == "output":
return sp.check_output(command_line, cwd = cwd)
else:
raise RuntimeError("Mode is expected to be one of 'raise', 'return_code' or 'output'")
def validate(context):
if shutil.which(context.config("java_binary")) == "":
raise RuntimeError("Cannot find Java binary at: %s" % context.config("java_binary"))
if not b"11" in sp.check_output([
shutil.which(context.config("java_binary")),
"-version"
], stderr = sp.STDOUT):
print("WARNING! A Java JDK of at least version 11 is recommended.")
def execute(context):
pass
\ No newline at end of file