Merge branch 'nfp_weekend' into nfpCovid19

Conflicts:
	synthesis/population/spatial/secondary/locations.py

config.yml

 # General pipeline settings
-working_directory: /nas/kaghog/nfp/cache_output_m11_sat
-flowchart_path: /nas/kaghog/nfp/cache_output_m11_sat/flowchart.json
+working_directory: &DIR /nas/cache_output
+flowchart_path: /nas/cache_output/flowchart.json
 dryrun: false
 
 # Requested stages
@@ -8,19 +8,26 @@ run:
  # - data.statpop.persons
  # - data.statpop.projections.households
  # - data.statpop.scaled
- # - synthesis.population.matched
+#  - synthesis.population.matched
  # - data.microcensus.households
  # - data.microcensus.csv
  # - population.destinations
  # - synthesis.population.destinations
+ # - synthesis.population.spatial.primary.weekend.pry_distance_distributions
+#  -  synthesis.population.spatial.primary.weekend.work_locations
+#  -  synthesis.population.spatial.primary.weekend.education_locations
+# - synthesis.population.spatial.secondary.distance_distributions
  # - synthesis.population.spatial.secondary.locations
  # - synthesis.population.spatial.primary.weekend.locations
- # - matsim.facilities
- # - matsim.population
- # - matsim.households
- # - matsim.run
-  - population.output
- # - analysis.analysis
+ # - synthesis.population.spatial.primary.locations
+ #- synthesis.population.spatial.locations
+ # - matsim.scenario.population
+ # - matsim.scenario.households
+ # - matsim.scenario.facilities
+  - matsim.simulation.run
+ # - matsim.runtime.eqasim
+ # - population.output
+ # - analysis.analysis_weekend
 
 # These are configuration options that we use in the pipeline
 config:
@@ -33,10 +40,14 @@ config:
     enable_scaling: true
     scaling_year: 2020
     use_freight: false
-    use_detailed_activities: false
-    weekend_scenario: true
-    specific_weekend_scenario: saturday
+    use_detailed_activities: false  #for a matsim run see eqasim_branch and repo below
+    weekend_scenario: false
+    specific_day_scenario: avgworkday
+    specific_weekend_scenario:
     hafas_date: 01.10.2018
     data_path: /nas/ivtmatsim/scenarios/switzerland/data
-    output_path: /nas/kaghog/nfp/cache_output_m11_sat
-    analysis_path: /nas/kaghog/nfp/cache_output_m11_sat/analysis
\ No newline at end of file
+    output_path: *DIR
+    analysis_path: /nas/cache_output/analysis
+    #eqasim_repository: https://github.com/kaghog/eqasim-java.git
+    #eqasim_branch: develop
+    #for eqasim version 1.3.1 only the develop branch allows for custom activities
\ No newline at end of file

config_grace.yml

 # General pipeline settings
-working_directory: /nas/kaghog/nfp/cache_output_m11_sun_25p
-flowchart_path: /nas/kaghog/nfp/cache_output_m11_sun_25p/flowchart.json
+working_directory: /nas/kaghog/nfp/cache_output_m11_mon
+flowchart_path: /nas/kaghog/nfp/cache_output_m11_mon/flowchart.json
 dryrun: false
 
 # Requested stages
@@ -8,7 +8,7 @@ run:
  # - data.statpop.persons
  # - data.statpop.projections.households
  # - data.statpop.scaled
- # - synthesis.population.matched
+#  - synthesis.population.matched
  # - data.microcensus.households
  # - data.microcensus.csv
  # - population.destinations
@@ -16,14 +16,15 @@ run:
  # - synthesis.population.spatial.primary.weekend.pry_distance_distributions
 #  -  synthesis.population.spatial.primary.weekend.work_locations
 #  -  synthesis.population.spatial.primary.weekend.education_locations
-#  - synthesis.population.spatial.secondary.distance_distributions
+# - synthesis.population.spatial.secondary.distance_distributions
 #  - synthesis.population.spatial.secondary.locations
- # - synthesis.population.spatial.primary.weekend.locations
+#  - synthesis.population.spatial.primary.weekend.locations
+#  - synthesis.population.spatial.primary.locations
  # - matsim.facilities
  # - matsim.population
  # - matsim.households
  # - matsim.run
-  - population.output
+ # - population.output
  # - analysis.analysis
 
 # These are configuration options that we use in the pipeline
@@ -33,14 +34,15 @@ config:
     hot_deck_matching_runners: 2
     disable_progress_bar: false
     java_memory: 80G
-    input_downsampling: 0.25
+    input_downsampling: 0.1
     enable_scaling: true
     scaling_year: 2020
     use_freight: false
-    use_detailed_activities: false
-    weekend_scenario: true
-    specific_weekend_scenario: sunday
+    use_detailed_activities: true
+    weekend_scenario: false
+    specific_day_scenario: monday #avgworkday
+    specific_weekend_scenario: saturday
     hafas_date: 01.10.2018
     data_path: /nas/ivtmatsim/scenarios/switzerland/data
-    output_path: /nas/kaghog/nfp/cache_output_m11_sun_25p
-    analysis_path: /nas/kaghog/nfp/cache_output_m11_sun_25p/analysis
\ No newline at end of file
+    output_path: /nas/kaghog/nfp/cache_output_m11_mon
+    analysis_path: /nas/kaghog/nfp/cache_output_m11_mon/analysis
\ No newline at end of file

data/microcensus/persons.py

@@ -63,6 +63,16 @@ def execute(context):
     df_mz_persons["sunday"] = False
     df_mz_persons.loc[df_mz_persons["tag"] == 7, "sunday"] = True
 
+    #specify day of the week
+    df_mz_persons["day"] = "monday"
+    df_mz_persons.loc[df_mz_persons["tag"] == 2, "day"] = "tuesday"
+    df_mz_persons.loc[df_mz_persons["tag"] == 3, "day"] = "wednesday"
+    df_mz_persons.loc[df_mz_persons["tag"] == 4, "day"] = "thursday"
+    df_mz_persons.loc[df_mz_persons["tag"] == 5, "day"] = "friday"
+    df_mz_persons.loc[df_mz_persons["tag"] == 6, "day"] = "saturday"
+    df_mz_persons.loc[df_mz_persons["tag"] == 7, "day"] = "sunday"
+
+
     # Here we extract a bit more than Kirill, but most likely it will be useful later
 
     df_mz_persons["subscriptions_ga"] = df_mz_persons["f41610a"] == 1
@@ -123,7 +133,7 @@ def execute(context):
         "subscriptions_verbund_class",
         "subscriptions_strecke_class",
         "age_class", "person_weight",
-        "weekend", "saturday", "sunday", "date"
+        "weekend", "saturday", "sunday", "date", "day"
     ]]
 
     # Merge in the other data sets

matsim/final.py

@@ -3,44 +3,80 @@ import shutil
 
 
 def configure(context):
-    context.stage("matsim.run")
+    context.stage("matsim.simulation.run")
+    context.stage("matsim.simulation.prepare")
+    context.stage("matsim.runtime.eqasim")
+
     context.config("output_path")
+    context.config("output_prefix", "switzerland_")
+    context.config("write_jar", True)
     context.config("output_id")
-    context.stage("contracts.contracts")
-    context.stage("population.output")
-
-def execute(context):
-    results_path = context.stage("matsim.run")
 
-    output_path = context.config("output_path")
-    output_id = context.config("output_id")
-
-    if not os.path.isdir(output_path):
-        raise RuntimeError("Output path does not exist:", output_path)
-
-    target_path = "%s/%s" % (output_path, output_id)
+    context.stage("population.output")
 
-    if os.path.exists(target_path):
-        if os.path.isdir(target_path):
-            print("Cleaning target directory:", target_path)
-            shutil.rmtree(target_path)
-        else:
-            raise RuntimeError("Cannot clean target path:", target_path)
+    #context.stage("documentation.meta_output")
+    context.stage("contracts.contracts")
 
-    os.mkdir(target_path)
+def execute(context):
+    config_path = "%s/%s" % (
+        context.path("matsim.simulation.prepare"),
+        context.stage("matsim.simulation.prepare")
+    )
 
-    for file in [
-        "switzerland_network.xml.gz",
-        "switzerland_transit_schedule.xml.gz",
-        "switzerland_transit_vehicles.xml.gz",
-        "switzerland_facilities.xml.gz",
-        "switzerland_households.xml.gz",
-        "switzerland_population.xml.gz",
-        "switzerland_config.xml"
+    for name in [
+        "%shouseholds.xml.gz" % context.config("output_prefix"),
+        "%spopulation.xml.gz" % context.config("output_prefix"),
+        "%sfacilities.xml.gz" % context.config("output_prefix"),
+        "%snetwork.xml.gz" % context.config("output_prefix"),
+        "%stransit_schedule.xml.gz" % context.config("output_prefix"),
+        "%stransit_vehicles.xml.gz" % context.config("output_prefix"),
+        "%sconfig.xml" % context.config("output_prefix")
     ]:
-        shutil.copyfile("%s/%s" % (results_path, file), "%s/%s" % (target_path, file))
+        shutil.copy(
+            "%s/%s" % (context.path("matsim.simulation.prepare"), name),
+            "%s/%s" % (context.config("output_path"), name)
+        )
 
-    contracts_path = context.stage("contracts.contracts")
-    shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % target_path)
+    if context.config("write_jar"):
+        shutil.copy(
+            "%s/%s" % (context.path("matsim.runtime.eqasim"), context.stage("matsim.runtime.eqasim")),
+            "%s/%srun.jar" % (context.config("output_path"), context.config("output_prefix"))
+        )
+        contracts_path = context.stage("contracts.contracts")
+        shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % (context.config("output_path"), context.config("output_prefix")))
 
-    return {}
+# def execute(context):
+#     results_path = context.stage("matsim.run")
+#
+#     output_path = context.config("output_path")
+#     output_id = context.config("output_id")
+#
+#     if not os.path.isdir(output_path):
+#         raise RuntimeError("Output path does not exist:", output_path)
+#
+#     target_path = "%s/%s" % (output_path, output_id)
+#
+#     if os.path.exists(target_path):
+#         if os.path.isdir(target_path):
+#             print("Cleaning target directory:", target_path)
+#             shutil.rmtree(target_path)
+#         else:
+#             raise RuntimeError("Cannot clean target path:", target_path)
+#
+#     os.mkdir(target_path)
+#
+#     for file in [
+#         "switzerland_network.xml.gz",
+#         "switzerland_transit_schedule.xml.gz",
+#         "switzerland_transit_vehicles.xml.gz",
+#         "switzerland_facilities.xml.gz",
+#         "switzerland_households.xml.gz",
+#         "switzerland_population.xml.gz",
+#         "switzerland_config.xml"
+#     ]:
+#         shutil.copyfile("%s/%s" % (results_path, file), "%s/%s" % (target_path, file))
+#
+#     contracts_path = context.stage("contracts.contracts")
+#     shutil.copyfile(contracts_path, "%s/CONTRACTS.html" % target_path)
+#
+#     return {}

matsim/java/eqasim.py

-import subprocess as sp
-
-
-def configure(context):
-    context.stage("utils.java")
-
-
-def execute(context):
-    java = context.stage("utils.java")
-
-    sp.check_call([
-        "git", "clone", "https://github.com/eqasim-org/eqasim-java.git"
-    ], cwd=context.cache_path)
-
-    sp.check_call([
-        "git", "checkout", "v1.2.1"
-    ], cwd="%s/eqasim-java" % context.cache_path)
-
-    sp.check_call([
-        "mvn", "-Pstandalone", "package"
-    ], cwd="%s/eqasim-java" % context.cache_path)
-
-    jar = "%s/eqasim-java/switzerland/target/switzerland-1.2.1.jar" % context.cache_path
-    return jar

matsim/java/pt2matsim.py

-import os
-import subprocess as sp
-
-
-def configure(context):
-    context.stage("utils.java")
-
-
-def execute(context):
-    java = context.stage("utils.java")
-
-    os.mkdir("%s/java_tmp" % context.cache_path)
-
-    sp.check_call([
-        "git", "clone", "https://github.com/matsim-org/pt2matsim.git"
-    ], cwd=context.cache_path)
-
-    sp.check_call([
-        "git", "checkout", "v19.10"
-    ], cwd="%s/pt2matsim" % context.cache_path)
-
-    sp.check_call([
-        "mvn", "-Djava.io.tmpdir=%s/java_tmp" % context.cache_path, "package"
-    ], cwd="%s/pt2matsim" % context.cache_path)
-
-    jar = "%s/pt2matsim/target/pt2matsim-19.10-shaded.jar" % context.cache_path
-    java(jar, "org.matsim.pt2matsim.run.CreateDefaultOsmConfig", ["test_config.xml"], cwd=context.cache_path)
-
-    assert (os.path.exists("%s/test_config.xml" % context.cache_path))
-    assert (os.path.exists("%s/java_tmp/GeoTools" % context.cache_path))
-
-    return jar, "%s/java_tmp" % context.cache_path

matsim/mz/population.py

@@ -110,7 +110,7 @@ def execute(context):
 
                         person_writer.write(writer)
                         number_of_written_persons += 1
-                        progress.update()
+                        #progress.update()
                 except StopIteration:
                     pass
 

matsim/run.py

-import shutil
-
-
-def configure(context):
-    context.stage("matsim.population")
-    context.stage("matsim.households")
-    context.stage("matsim.facilities")
-    context.stage("matsim.network.mapped")
-    context.stage("matsim.java.eqasim")
-    context.stage("utils.java")
-
-
-def execute(context):
-    # Some files we just copy
-    transit_vehicles_path = context.stage("matsim.network.mapped")["vehicles"]
-    shutil.copyfile(transit_vehicles_path, "%s/switzerland_transit_vehicles.xml.gz" % context.cache_path)
-
-    households_path = context.stage("matsim.households")
-    shutil.copyfile(households_path, "%s/switzerland_households.xml.gz" % context.cache_path)
-
-    # Some files we send through the preparation script
-    transit_schedule_input_path = context.stage("matsim.network.mapped")["schedule"]
-    transit_schedule_output_path = "%s/switzerland_transit_schedule.xml.gz" % context.cache_path
-
-    network_input_path = context.stage("matsim.network.mapped")["network"]
-    network_output_path = "%s/switzerland_network.xml.gz" % context.cache_path
-
-    facilities_input_path = context.stage("matsim.facilities")
-    facilities_output_path = "%s/switzerland_facilities.xml.gz" % context.cache_path
-
-    population_input_path = context.stage("matsim.population")
-    population_prepared_path = "%s/prepared_population.xml.gz" % context.cache_path
-    population_output_path = "%s/switzerland_population.xml.gz" % context.cache_path
-
-    config_output_path = "%s/switzerland_config.xml" % context.cache_path
-
-    # Call preparation script
-    java = context.stage("utils.java")
-
-    java(
-        context.stage("matsim.java.eqasim"),
-        "org.eqasim.core.scenario.preparation.RunPreparation", [
-            "--input-facilities-path", facilities_input_path,
-            "--output-facilities-path", facilities_output_path,
-            "--input-population-path", population_input_path,
-            "--output-population-path", population_prepared_path,
-            "--input-network-path", network_input_path,
-            "--output-network-path", network_output_path,
-            "--threads", str(context.config("threads"))
-        ], cwd=context.cache_path)
-
-    java(
-        context.stage("matsim.java.eqasim"),
-        "org.eqasim.core.scenario.config.RunGenerateConfig", [
-            "--output-path", config_output_path,
-            "--prefix", "switzerland_",
-            "--sample-size", str(context.config("input_downsampling")),
-            "--random-seed", str(1000),
-            "--threads", str(context.config("threads"))
-        ], cwd=context.cache_path)
-
-    java(
-        context.stage("matsim.java.eqasim"),
-        "org.eqasim.switzerland.scenario.RunCalculateStopCategories", [
-            "--input-path", transit_schedule_input_path,
-            "--output-path", transit_schedule_output_path
-        ], cwd=context.cache_path)
-
-    java(
-        context.stage("matsim.java.eqasim"),
-        "org.eqasim.core.scenario.routing.RunPopulationRouting", [
-            "--config-path", config_output_path,
-            "--output-path", population_output_path,
-            "--threads", str(context.config("threads")),
-            "--config:plans.inputPlansFile", population_prepared_path
-        ], cwd=context.cache_path)
-
-    java(
-        context.stage("matsim.java.eqasim"),
-        "org.eqasim.core.scenario.validation.RunScenarioValidator", [
-            "--config-path", config_output_path
-        ], cwd=context.cache_path)
-
-    java(
-        context.stage("matsim.java.eqasim"),
-        "org.eqasim.switzerland.scenario.RunAdaptConfig", [
-            "--input-path", config_output_path,
-            "--output-path", config_output_path
-        ], cwd=context.cache_path)
-
-    java(
-        context.stage("matsim.java.eqasim"),
-        "org.eqasim.switzerland.RunSimulation", [
-            "--config-path", config_output_path,
-            "--config:controler.lastIteration", str(1),
-            "--config:controler.writeEventsInterval", str(1),
-            "--config:controler.writePlansInterval", str(1),
-        ], cwd=context.cache_path)
-
-    return context.cache_path

matsim/runtime/eqasim.py

+import subprocess as sp
+import os, os.path, shutil
+
+import matsim.runtime.git as git
+import matsim.runtime.java as java
+import matsim.runtime.maven as maven
+
+def configure(context):
+    context.stage("matsim.runtime.git")
+    context.stage("matsim.runtime.java")
+    context.stage("matsim.runtime.maven")
+
+    context.config("eqasim_version", "1.3.1")
+    context.config("eqasim_branch", "v1.3.1")
+    context.config("eqasim_repository", "https://github.com/eqasim-org/eqasim-java.git")
+    context.config("eqasim_path", "")
+
+def run(context, command, arguments):
+    version = context.config("eqasim_version")
+
+    # Make sure there is a dependency
+    context.stage("matsim.runtime.eqasim")
+
+    jar_path = "%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (
+        context.path("matsim.runtime.eqasim"), version
+    )
+    java.run(context, command, arguments, jar_path)
+
+def execute(context):
+    version = context.config("eqasim_version")
+    print("THIS IS A CHECK!!!")
+    print(context.config("eqasim_repository"))
+
+    # Normal case: we clone eqasim
+    if context.config("eqasim_path") == "":
+        # Clone repository and checkout version
+        git.run(context, [
+            "clone", context.config("eqasim_repository"),
+            "--branch", context.config("eqasim_branch"),
+            "--single-branch", "eqasim-java",
+            "--depth", "1"
+        ])
+
+        # Build eqasim
+        maven.run(context, ["-Pstandalone", "--projects", "switzerland", "--also-make", "package"], cwd = "%s/eqasim-java" % context.path())
+        jar_path = "%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (context.path(), version)
+
+    # Special case: We provide the jar directly. This is mainly used for
+    # creating input to unit tests of the eqasim-java package.
+    else:
+        os.makedirs("%s/eqasim-java/switzerland/target" % context.path())
+        shutil.copy(context.config("eqasim_path"),
+            "%s/eqasim-java/switzerland/target/switzerland-%s.jar" % (context.path(), version))
+
+    return "eqasim-java/switzerland/target/switzerland-%s.jar" % version
+
+def validate(context):
+    path = context.config("eqasim_path")
+
+    if path == "":
+        return True
+
+    if not os.path.exists(path):
+        raise RuntimeError("Cannot find eqasim at: %s" % path)
+
+    return os.path.getmtime(path)
\ No newline at end of file

matsim/runtime/git.py

+import subprocess as sp
+import shutil
+
+def configure(context):
+    context.config("git_binary", "git")
+
+def run(context, arguments = [], cwd = None, catch_output = False):
+    """
+        This function calls git.
+    """
+    # Make sure there is a dependency
+    context.stage("matsim.runtime.git")
+
+    if cwd is None:
+        cwd = context.path()
+
+    command_line = [
+        shutil.which(context.config("git_binary"))
+    ] + arguments
+
+    if catch_output:
+        return sp.check_output(command_line, cwd = cwd).decode("utf-8").strip()
+
+    else:
+        return_code = sp.check_call(command_line, cwd = cwd)
+
+        if not return_code == 0:
+            raise RuntimeError("Git return code: %d" % return_code)
+
+def validate(context):
+    if shutil.which(context.config("git_binary")) == "":
+        raise RuntimeError("Cannot find git binary at: %s" % context.config("git_binary"))
+
+    if not b"2." in sp.check_output([
+        shutil.which(context.config("git_binary")),
+        "--version"
+    ], stderr = sp.STDOUT):
+        print("WARNING! Git of at least version 2.x.x is recommended!")
+
+def execute(context):
+    pass
\ No newline at end of file

matsim/runtime/java.py

+import subprocess as sp
+import os, shutil
+
+def configure(context):
+    context.config("java_binary", "java")
+    context.config("java_memory", "50G")
+
+def run(context, entry_point, arguments = [], class_path = None, vm_arguments = [], cwd = None, memory = None, mode = "raise"):
+    """
+        This function calls java code. There are three modes:
+        - return_code: Returns the return code of the Java call
+        - output: Returns the output of the Java call
+        - raise (default): Raises an exception if the return code is not zero
+    """
+    # Make sure there is a dependency
+    context.stage("matsim.runtime.java")
+
+    # Prepare temp folder
+    temp_path = "%s/__java_temp" % context.path()
+    if not os.path.exists(temp_path):
+        os.mkdir(temp_path)
+
+    # Prepare arguments
+    memory = context.config("java_memory") if memory is None else memory
+    vm_arguments = [
+        "-Xmx" + memory,
+        "-Djava.io.tmpdir=%s" % temp_path,
+        "-Dmatsim.useLocalDtds=true"
+    ] + vm_arguments
+
+    # Prepare classpath
+    if type(class_path) == list or type(class_path) == tuple:
+        class_path = ":".join(class_path)
+
+    # Prepare CWD
+    if cwd is None:
+        cwd = context.path()
+
+    # Prepare command line
+    command_line = [
+        shutil.which(context.config("java_binary")),
+        "-cp", class_path
+    ] + vm_arguments + [
+        entry_point
+    ] + arguments
+
+    command_line = list(map(str, command_line))
+
+    print("Executing java:", " ".join(command_line))
+
+    if mode == "raise" or mode == "return_code":
+        return_code = sp.check_call(command_line, cwd = cwd)
+
+        if not return_code == 0:
+            raise RuntimeError("Java return code: %d" % return_code)
+
+        return return_code
+    elif mode == "output":
+        return sp.check_output(command_line, cwd = cwd)
+    else:
+        raise RuntimeError("Mode is expected to be one of 'raise', 'return_code' or 'output'")
+
+def validate(context):
+    if shutil.which(context.config("java_binary")) == "":
+        raise RuntimeError("Cannot find Java binary at: %s" % context.config("java_binary"))
+
+    if not b"11" in sp.check_output([
+        shutil.which(context.config("java_binary")),
+        "-version"
+    ], stderr = sp.STDOUT):
+        print("WARNING! A Java JDK of at least version 11 is recommended.")
+
+def execute(context):
+    pass
\ No newline at end of file