Merge pull request #399 from mithro/litex-sm2py

Converting LiteX to use Python modules.

.gitmodules

@@ -1,30 +0,0 @@
-[submodule "litex/soc/cores/cpu/lm32/verilog/submodule"]
-	path = litex/soc/cores/cpu/lm32/verilog/submodule
-	url = https://github.com/m-labs/lm32.git
-[submodule "litex/soc/cores/cpu/mor1kx/verilog"]
-	path = litex/soc/cores/cpu/mor1kx/verilog
-	url = https://github.com/openrisc/mor1kx.git
-[submodule "litex/soc/software/compiler_rt"]
-	path = litex/soc/software/compiler_rt
-	url = https://github.com/llvm-mirror/compiler-rt
-[submodule "litex/soc/cores/cpu/picorv32/verilog"]
-	path = litex/soc/cores/cpu/picorv32/verilog
-	url = https://github.com/cliffordwolf/picorv32
-[submodule "litex/build/sim/core/modules/ethernet/tapcfg"]
-	path = litex/build/sim/core/modules/ethernet/tapcfg
-	url = https://github.com/enjoy-digital/tapcfg
-[submodule "litex/soc/cores/cpu/vexriscv/verilog"]
-	path = litex/soc/cores/cpu/vexriscv/verilog
-	url = https://github.com/enjoy-digital/VexRiscv-verilog.git
-[submodule "litex/soc/cores/cpu/minerva/verilog"]
-	path = litex/soc/cores/cpu/minerva/verilog
-	url = https://github.com/lambdaconcept/minerva
-[submodule "litex/soc/cores/cpu/rocket/verilog"]
-	path = litex/soc/cores/cpu/rocket/verilog
-	url = https://github.com/enjoy-digital/rocket-litex-verilog
-[submodule "litex/soc/cores/cpu/microwatt/sources"]
-	path = litex/soc/cores/cpu/microwatt/sources
-	url = https://github.com/antonblanchard/microwatt
-[submodule "litex/soc/cores/cpu/blackparrot/pre-alpha-release"]
-	path = litex/soc/cores/cpu/blackparrot/pre-alpha-release
-	url = https://github.com/enjoy-digital/black-parrot.git

MANIFEST.in

@@ -1,8 +1 @@
 graft litex/build/sim
-graft litex/soc/software
-graft litex/soc/cores/cpu/lm32/verilog
-graft litex/soc/cores/cpu/minerva/verilog
-graft litex/soc/cores/cpu/mor1kx/verilog
-graft litex/soc/cores/cpu/picorv32/verilog
-graft litex/soc/cores/cpu/rocket/verilog
-graft litex/soc/cores/cpu/vexriscv/verilog

litex/__init__.py

@@ -9,3 +9,20 @@ from litex.soc.integration import export
 sys.modules["litex.soc.integration.cpu_interface"] = export
 
 from litex.tools.litex_client import RemoteClient
+
+def get_data_mod(data_type, data_name):
+    """Get the pythondata-{}-{} module or raise a useful error message."""
+    imp = "import pythondata_{}_{} as dm".format(data_type, data_name)
+    try:
+        l = {}
+        exec(imp, {}, l)
+        dm = l['dm']
+        return dm
+    except ImportError as e:
+        raise ImportError("""\
+pythondata-{dt}-{dn} module not installed! Unable to use {dn} {dt}.
+{e}
+
+You can install this by running;
+ pip install git+https://github.com/litex-hub/pythondata-{dt}-{dn}.git
+""".format(dt=data_type, dn=data_name, e=e))

litex/build/sim/core/modules/ethernet/Makefile

@@ -3,7 +3,7 @@ UNAME_S := $(shell uname -s)
 
 include $(SRC_DIR)/modules/rules.mak
 
-CFLAGS += -I$(MOD_SRC_DIR)/tapcfg/src/include
+CFLAGS += -I$(TAPCFG_DIRECTORY)/src/include
 OBJS = $(MOD).o tapcfg.o taplog.o
 
 $(MOD).so: $(OBJS)
@@ -13,8 +13,8 @@ else
 	$(CC) $(LDFLAGS) -Wl,-soname,$@ -o $@ $^
 endif
 
-tapcfg.o: $(MOD_SRC_DIR)/tapcfg/src/lib/tapcfg.c
+tapcfg.o: $(TAPCFG_DIRECTORY)/src/lib/tapcfg.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 
-taplog.o: $(MOD_SRC_DIR)/tapcfg/src/lib/taplog.c
+taplog.o: $(TAPCFG_DIRECTORY)/src/lib/taplog.c
 	$(CC) $(CFLAGS) -c -o $@ $<

litex/build/sim/core/modules/ethernet/tapcfg

@@ -1 +0,0 @@
-Subproject commit bd557ff00d8fe2473fcf346e36c96d004e94b8ca

litex/build/sim/core/modules/xgmii_ethernet/Makefile

@@ -3,7 +3,7 @@ UNAME_S := $(shell uname -s)
 
 include $(SRC_DIR)/modules/rules.mak
 
-CFLAGS += -I$(MOD_SRC_DIR)/tapcfg/src/include
+CFLAGS += -I$(TAPCFG_DIRECTORY)/src/include
 OBJS = $(MOD).o tapcfg.o taplog.o
 
 $(MOD).so: $(OBJS)
@@ -13,8 +13,8 @@ else
 	$(CC) $(LDFLAGS) -Wl,-soname,$@ -o $@ $^
 endif
 
-tapcfg.o: $(MOD_SRC_DIR)/tapcfg/src/lib/tapcfg.c
+tapcfg.o: $(TAPCFG_DIRECTORY)/src/lib/tapcfg.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 
-taplog.o: $(MOD_SRC_DIR)/tapcfg/src/lib/taplog.c
+taplog.o: $(TAPCFG_DIRECTORY)/src/lib/taplog.c
 	$(CC) $(CFLAGS) -c -o $@ $<

litex/build/sim/core/modules/xgmii_ethernet/tapcfg

@@ -1 +0,0 @@
-../ethernet/tapcfg/

litex/build/sim/verilator.py

@@ -7,7 +7,7 @@ import sys
 import subprocess
 
 from migen.fhdl.structure import _Fragment
-
+from litex import get_data_mod
 from litex.build import tools
 from litex.build.generic_platform import *
 
@@ -102,13 +102,15 @@ extern "C" void litex_sim_init(void **out)
 
 
 def _generate_sim_variables(include_paths):
+    tapcfg_dir = get_data_mod("misc", "tapcfg").data_location
     include = ""
     for path in include_paths:
         include += "-I"+path+" "
     content = """\
 SRC_DIR = {}
 INC_DIR = {}
-""".format(core_directory, include)
+TAPCFG_DIRECTORY = {}
+""".format(core_directory, include, tapcfg_dir)
     tools.write_to_file("variables.mak", content)
 
 

litex/soc/cores/cpu/blackparrot/core.py

@@ -32,6 +32,7 @@ import os
 
 from migen import *
 
+from litex import get_data_mod
 from litex.soc.interconnect import axi
 from litex.soc.interconnect import wishbone
 from litex.soc.cores.cpu import CPU
@@ -116,7 +117,8 @@ class BlackParrotRV64(CPU):
 
     @staticmethod
     def add_sources(platform, variant="standard"):
-        filename = os.path.join(os.path.abspath(os.path.dirname(__file__)), "flist_litex.verilator")
+        filename = get_data_mod("cpu", "blackparrot").data_file(
+            "flist_litex.verilator")
         with open(filename) as openfileobject:
             for line in openfileobject:
                 temp = line

litex/soc/cores/cpu/blackparrot/pre-alpha-release

@@ -1 +0,0 @@
-Subproject commit dbb13f31370a743633dc94d3639d55c8c4d74e1d

litex/soc/cores/cpu/lm32/core.py

@@ -9,6 +9,7 @@ import os
 
 from migen import *
 
+from litex import get_data_mod
 from litex.soc.interconnect import wishbone
 from litex.soc.cores.cpu import CPU
 
@@ -97,9 +98,8 @@ class LM32(CPU):
 
     @staticmethod
     def add_sources(platform, variant):
-        vdir = os.path.join(
+        vdir = get_data_mod("cpu", "lm32").data_location
-            os.path.abspath(os.path.dirname(__file__)), "verilog")
+        platform.add_sources(os.path.join(vdir, "rtl"),
-        platform.add_sources(os.path.join(vdir, "submodule", "rtl"),
             "lm32_cpu.v",
             "lm32_instruction_unit.v",
             "lm32_decoder.v",
@@ -118,7 +118,7 @@ class LM32(CPU):
             "lm32_debug.v",
             "lm32_itlb.v",
             "lm32_dtlb.v")
-        platform.add_verilog_include_path(os.path.join(vdir, "submodule", "rtl"))
+        platform.add_verilog_include_path(os.path.join(vdir, "rtl"))
         if variant == "minimal":
             platform.add_verilog_include_path(os.path.join(vdir, "config_minimal"))
         elif variant == "lite":

litex/soc/cores/cpu/lm32/verilog/submodule

@@ -1 +0,0 @@
-Subproject commit 84b3e3ca0ad9535acaef201c1482342871358b08

litex/soc/cores/cpu/microwatt/core.py

@@ -6,6 +6,7 @@ import os
 
 from migen import *
 
+from litex import get_data_mod
 from litex.soc.interconnect import wishbone
 from litex.soc.cores.cpu import CPU
 
@@ -99,7 +100,9 @@ class Microwatt(CPU):
 
     @staticmethod
     def add_sources(platform):
-        sdir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sources")
+        sdir = os.path.join(
+            get_data_mod("cpu", "microwatt").data_location,
+            "sources")
         platform.add_sources(sdir,
             # Common / Types / Helpers
             "decode_types.vhdl",

litex/soc/cores/cpu/microwatt/sources

@@ -1 +0,0 @@
-Subproject commit 1a826f077bb518bc3ffd799c47a6dd2852165f89

litex/soc/cores/cpu/minerva/verilog

@@ -1 +0,0 @@
-Subproject commit fb296e4e48e5ced8dd05f2228d84b4bc18f54f75

litex/soc/cores/cpu/mor1kx/core.py

@@ -8,6 +8,7 @@ import os
 
 from migen import *
 
+from litex import get_data_mod
 from litex.soc.interconnect import wishbone
 from litex.soc.cores.cpu import CPU
 
@@ -175,8 +176,8 @@ class MOR1KX(CPU):
     @staticmethod
     def add_sources(platform):
         vdir = os.path.join(
-            os.path.abspath(os.path.dirname(__file__)),
+            get_data_mod("cpu", "mor1kx").data_location,
-            "verilog", "rtl", "verilog")
+            "rtl", "verilog")
         platform.add_source_dir(vdir)
         platform.add_verilog_include_path(vdir)
 

litex/soc/cores/cpu/mor1kx/verilog

@@ -1 +0,0 @@
-Subproject commit 69b97fcb43b35d6c6639ecc68e63d912c09ee8da

litex/soc/cores/cpu/picorv32/core.py

@@ -11,6 +11,7 @@ import os
 
 from migen import *
 
+from litex import get_data_mod
 from litex.soc.interconnect import wishbone
 from litex.soc.cores.cpu import CPU
 
@@ -180,8 +181,7 @@ class PicoRV32(CPU):
 
     @staticmethod
     def add_sources(platform):
-        vdir = os.path.join(
+        vdir = get_data_mod("cpu", "picorv32").data_location
-            os.path.abspath(os.path.dirname(__file__)), "verilog")
         platform.add_source(os.path.join(vdir, "picorv32.v"))
 
     def do_finalize(self):

litex/soc/cores/cpu/picorv32/verilog

@@ -1 +0,0 @@
-Subproject commit a9e0ea54cffa162cfe901ff8d30d8877a18c6d8e

litex/soc/cores/cpu/rocket/core.py

@@ -33,6 +33,7 @@ import os
 
 from migen import *
 
+from litex import get_data_mod
 from litex.soc.interconnect import axi
 from litex.soc.interconnect import wishbone
 from litex.soc.cores.cpu import CPU
@@ -239,8 +240,7 @@ class RocketRV64(CPU):
 
     @staticmethod
     def add_sources(platform, variant="standard"):
-        vdir = os.path.join(
+        vdir = get_data_mod("cpu", "rocket").data_location
-            os.path.abspath(os.path.dirname(__file__)), "verilog")
         platform.add_sources(
             os.path.join(vdir, "generated-src"),
             CPU_VARIANTS[variant] + ".v",

litex/soc/cores/cpu/rocket/verilog

@@ -1 +0,0 @@
-Subproject commit fb31001d9655ebfb8ab25209e094939f68feb6a7

litex/soc/cores/cpu/vexriscv/core.py

@@ -12,6 +12,7 @@ import os
 
 from migen import *
 
+from litex import get_data_mod
 from litex.soc.interconnect import wishbone
 from litex.soc.interconnect.csr import *
 from litex.soc.cores.cpu import CPU
@@ -249,7 +250,7 @@ class VexRiscv(CPU, AutoCSR):
     @staticmethod
     def add_sources(platform, variant="standard"):
         cpu_filename = CPU_VARIANTS[variant] + ".v"
-        vdir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "verilog")
+        vdir = get_data_mod("cpu", "vexriscv").data_location
         platform.add_source(os.path.join(vdir, cpu_filename))
 
     def use_external_variant(self, variant_filename):

litex/soc/cores/cpu/vexriscv/verilog

@@ -1 +0,0 @@
-Subproject commit 8baad219885a47f65959a9cd4870691e84678db4

litex/soc/integration/builder.py

@@ -14,6 +14,7 @@ import subprocess
 import struct
 import shutil
 
+from litex import get_data_mod
 from litex.build.tools import write_to_file
 from litex.soc.integration import export, soc_core
 
@@ -100,6 +101,9 @@ class Builder:
                 exec_profiles["EXECUTE_IN_PLACE"] = "1"
             for k, v in exec_profiles.items():
                 define(k, v)
+            define(
+                "COMPILER_RT_DIRECTORY",
+                get_data_mod("software", "compiler_rt").data_location)
             define("SOC_DIRECTORY", soc_directory)
             variables_contents.append("export BUILDINC_DIRECTORY\n")
             define("BUILDINC_DIRECTORY", self.include_dir)

litex/soc/software/compiler_rt

@@ -1 +0,0 @@
-Subproject commit 81fb4f00c2cfe13814765968e09931ffa93b5138

litex/soc/software/libcompiler_rt/Makefile

@@ -23,7 +23,7 @@ libcompiler_rt.a: $(OBJECTS)
 mulsi3.o: $(SOC_DIRECTORY)/software/libcompiler_rt/mulsi3.c
 	$(compile)
 
-%.o: $(SOC_DIRECTORY)/software/compiler_rt/lib/builtins/%.c
+%.o: $(COMPILER_RT_DIRECTORY)/lib/builtins/%.c
 	$(compile)
 
 .PHONY: all clean

litex_setup.py

@@ -8,7 +8,8 @@ from collections import OrderedDict
 
 import urllib.request
 
-current_path = os.path.dirname(os.path.realpath(__file__))
+current_path = os.path.abspath(os.curdir)
+
 
 # Repositories -------------------------------------------------------------------------------------
 
@@ -18,7 +19,8 @@ repos = [
     ("migen",        ("https://github.com/m-labs/",        True,  True)),
 
     # LiteX SoC builder
-    ("litex",        ("https://github.com/enjoy-digital/", True,  True)),
+    ("pythondata-software-compiler_rt", ("https://github.com/litex-hub/", False, True)),
+    ("litex",        ("https://github.com/enjoy-digital/", False,  True)),
 
     # LiteX cores ecosystem
     ("liteeth",      ("https://github.com/enjoy-digital/", False, True)),
@@ -34,6 +36,16 @@ repos = [
 
     # LiteX boards support
     ("litex-boards", ("https://github.com/litex-hub/",     False, True)),
+
+    # Optional LiteX data
+    ("pythondata-cpu-blackparrot", ("https://github.com/litex-hub/", False, True)),
+    ("pythondata-cpu-mor1kx",      ("https://github.com/litex-hub/", False, True)),
+    ("pythondata-cpu-lm32",        ("https://github.com/litex-hub/", False, True)),
+    ("pythondata-cpu-microwatt",   ("https://github.com/litex-hub/", False, True)),
+    ("pythondata-cpu-picorv32",    ("https://github.com/litex-hub/", False, True)),
+    ("pythondata-cpu-rocket",      ("https://github.com/litex-hub/", False, True)),
+    ("pythondata-cpu-vexriscv",    ("https://github.com/litex-hub/", False, True)),
+    ("pythondata-misc-tapcfg",     ("https://github.com/litex-hub/", False, True)),
 ]
 repos = OrderedDict(repos)
 
@@ -127,7 +139,7 @@ if "update" in sys.argv[1:]:
         print("[updating " + name + "]...")
         os.chdir(os.path.join(current_path, name))
         subprocess.check_call(
-            "git pull",
+            "git pull --ff-only",
             shell=True)
         os.chdir(os.path.join(current_path))
 

setup.py

@@ -14,9 +14,25 @@ setup(
     test_suite="test",
     license="BSD",
     python_requires="~=3.6",
-    install_requires=["migen", "pyserial", "requests"],
+    install_requires=[
+        "migen",
+        "pyserial",
+        "requests",
+        "pythondata-software-compiler_rt",
+    ],
     packages=find_packages(exclude=("test*", "sim*", "doc*")),
     include_package_data=True,
+    platforms=["Any"],
+    keywords="HDL ASIC FPGA hardware design",
+    classifiers=[
+        "Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)",
+        "Environment :: Console",
+        "Development Status :: Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+    ],
     entry_points={
         "console_scripts": [
             # full names