Skip to content

Commit

Permalink
Merge pull request ARMmbed#2179 from sarahmarshy/progen_build_tests_v2
Browse files Browse the repository at this point in the history
IDE build tests with progen
  • Loading branch information
screamerbg committed Jul 16, 2016
2 parents 7251a8c + b969fa5 commit 4e1ac4d
Show file tree
Hide file tree
Showing 9 changed files with 336 additions and 73 deletions.
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ PySerial>=2.7
PrettyTable>=0.7.2
Jinja2>=2.7.3
IntelHex>=1.3
project-generator>=0.9.3,<0.10.0
project-generator>=0.9.7,<0.10.0
project_generator_definitions>=0.2.26,<0.3.0
junit-xml
pyYAML
Expand Down
31 changes: 22 additions & 9 deletions tools/export/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

from tools.utils import mkdir
from tools.export import uvision4, uvision5, codered, gccarm, ds5_5, iar, emblocks, coide, kds, zip, simplicityv3, atmelstudio, sw4stm32, e2studio
from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException
from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException, FailedBuildException
from tools.targets import TARGET_NAMES, EXPORT_MAP, TARGET_MAP

from project_generator_definitions.definitions import ProGenDef
Expand Down Expand Up @@ -58,7 +58,8 @@ def online_build_url_resolver(url):


def export(project_path, project_name, ide, target, destination='/tmp/',
tempdir=None, clean=True, extra_symbols=None, make_zip=True, sources_relative=False, build_url_resolver=online_build_url_resolver):
tempdir=None, pgen_build = False, clean=True, extra_symbols=None, make_zip=True, sources_relative=False,
build_url_resolver=online_build_url_resolver, progen_build=False):
# Convention: we are using capitals for toolchain and target names
if target is not None:
target = target.upper()
Expand All @@ -68,8 +69,8 @@ def export(project_path, project_name, ide, target, destination='/tmp/',

use_progen = False
supported = True
report = {'success': False, 'errormsg':''}
report = {'success': False, 'errormsg':'', 'skip': False}

if ide is None or ide == "zip":
# Simple ZIP exporter
try:
Expand All @@ -83,6 +84,7 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
else:
if ide not in EXPORTERS:
report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
report['skip'] = True
else:
Exporter = EXPORTERS[ide]
target = EXPORT_MAP.get(target, target)
Expand All @@ -91,24 +93,35 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
use_progen = True
except AttributeError:
pass

if target not in Exporter.TARGETS or Exporter.TOOLCHAIN not in TARGET_MAP[target].supported_toolchains:
supported = False

if use_progen:
if not ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']):
supported = False
else:
if target not in Exporter.TARGETS:
supported = False

if supported:
# target checked, export
try:
exporter = Exporter(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols, sources_relative=sources_relative)
exporter.scan_and_copy_resources(project_path, tempdir, sources_relative)
exporter.generate()
report['success'] = True
if progen_build:
#try to build with pgen ide builders
try:
exporter.generate(progen_build=True)
report['success'] = True
except FailedBuildException, f:
report['errormsg'] = "Build Failed"
else:
exporter.generate()
report['success'] = True
except OldLibrariesException, e:
report['errormsg'] = ERROR_MESSAGE_NOT_EXPORT_LIBS

else:
report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
report['skip'] = True

zip_path = None
if report['success']:
Expand Down
9 changes: 8 additions & 1 deletion tools/export/exporters.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@

class OldLibrariesException(Exception): pass

class FailedBuildException(Exception) : pass

class Exporter(object):
TEMPLATE_DIR = dirname(__file__)
DOT_IN_RELATIVE_PATH = False
Expand Down Expand Up @@ -107,14 +109,19 @@ def progen_get_project_data(self):
}
return project_data

def progen_gen_file(self, tool_name, project_data):
def progen_gen_file(self, tool_name, project_data, progen_build=False):
""" Generate project using ProGen Project API """
settings = ProjectSettings()
project = Project(self.program_name, [project_data], settings)
# TODO: Fix this, the inc_dirs are not valid (our scripts copy files), therefore progen
# thinks it is not dict but a file, and adds them to workspace.
project.project['common']['include_paths'] = self.resources.inc_dirs
project.generate(tool_name, copied=not self.sources_relative)
if progen_build:
print("Project exported, building...")
result = project.build(tool_name)
if result == -1:
raise FailedBuildException("Build Failed")

def __scan_all(self, path):
resources = []
Expand Down
7 changes: 5 additions & 2 deletions tools/export/iar.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def TARGETS(self):
continue
return self._targets_supported

def generate(self):
def generate(self, progen_build=False):
""" Generates the project files """
project_data = self.progen_get_project_data()
tool_specific = {}
Expand Down Expand Up @@ -78,7 +78,10 @@ def generate(self):
# VLA is enabled via template IccAllowVLA
project_data['tool_specific']['iar']['misc']['c_flags'].remove("--vla")
project_data['common']['build_dir'] = os.path.join(project_data['common']['build_dir'], 'iar_arm')
self.progen_gen_file('iar_arm', project_data)
if progen_build:
self.progen_gen_file('iar_arm', project_data, True)
else:
self.progen_gen_file('iar_arm', project_data)

# Currently not used, we should reuse folder_name to create virtual folders
class IarFolder():
Expand Down
7 changes: 5 additions & 2 deletions tools/export/uvision4.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def TARGETS(self):
def get_toolchain(self):
return TARGET_MAP[self.target].default_toolchain

def generate(self):
def generate(self, progen_build=False):
""" Generates the project files """
project_data = self.progen_get_project_data()
tool_specific = {}
Expand Down Expand Up @@ -99,4 +99,7 @@ def generate(self):
i += 1
project_data['common']['macros'].append('__ASSERT_MSG')
project_data['common']['build_dir'] = join(project_data['common']['build_dir'], 'uvision4')
self.progen_gen_file('uvision', project_data)
if progen_build:
self.progen_gen_file('uvision', project_data, True)
else:
self.progen_gen_file('uvision', project_data)
7 changes: 5 additions & 2 deletions tools/export/uvision5.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def TARGETS(self):
def get_toolchain(self):
return TARGET_MAP[self.target].default_toolchain

def generate(self):
def generate(self, progen_build=False):
""" Generates the project files """
project_data = self.progen_get_project_data()
tool_specific = {}
Expand Down Expand Up @@ -102,4 +102,7 @@ def generate(self):
project_data['common']['macros'].pop(i)
i += 1
project_data['common']['macros'].append('__ASSERT_MSG')
self.progen_gen_file('uvision5', project_data)
if progen_build:
self.progen_gen_file('uvision5', project_data, True)
else:
self.progen_gen_file('uvision5', project_data)
66 changes: 10 additions & 56 deletions tools/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,14 @@
from argparse import ArgumentParser
from os import path

from tools.paths import EXPORT_DIR, EXPORT_WORKSPACE, EXPORT_TMP
from tools.paths import MBED_BASE, MBED_LIBRARIES
from tools.export import export, setup_user_prj, EXPORTERS, mcu_ide_matrix
from tools.utils import args_error, mkdir
from tools.tests import TESTS, Test, TEST_MAP
from tools.paths import EXPORT_DIR
from tools.export import export, EXPORTERS, mcu_ide_matrix
from tools.tests import TESTS, TEST_MAP
from tools.tests import test_known, test_name_known
from tools.targets import TARGET_NAMES
from tools.libraries import LIBRARIES
from utils import argparse_lowercase_type, argparse_uppercase_type, argparse_filestring_type, argparse_many
from utils import argparse_filestring_type, argparse_many
from utils import argparse_force_lowercase_type, argparse_force_uppercase_type
from project_api import setup_project, perform_export, print_results, get_lib_symbols



Expand Down Expand Up @@ -129,56 +127,20 @@
# Export results
successes = []
failures = []
zip = True
clean = True

# source_dir = use relative paths, otherwise sources are copied
sources_relative = True if options.source_dir else False

for mcu in options.mcu:
# Program Number or name
p, src, ide = options.program, options.source_dir, options.ide
project_dir, project_name, project_temp = setup_project(mcu, ide, p, src, options.build)

if src:
# --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
project_dir = options.source_dir
project_name = TESTS[p] if p else "Unnamed_project"
project_temp = path.join(options.source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu))
mkdir(project_temp)
lib_symbols = []
if options.macros:
lib_symbols += options.macros
zip = False # don't create zip
clean = False # don't cleanup because we use the actual source tree to generate IDE files
else:
test = Test(p)

# Some libraries have extra macros (called by exporter symbols) to we need to pass
# them to maintain compilation macros integrity between compiled library and
# header files we might use with it
lib_symbols = []
if options.macros:
lib_symbols += options.macros
for lib in LIBRARIES:
if lib['build_dir'] in test.dependencies:
lib_macros = lib.get('macros', None)
if lib_macros is not None:
lib_symbols.extend(lib_macros)

if not options.build:
# Substitute the library builds with the sources
# TODO: Substitute also the other library build paths
if MBED_LIBRARIES in test.dependencies:
test.dependencies.remove(MBED_LIBRARIES)
test.dependencies.append(MBED_BASE)

# Build the project with the same directory structure of the mbed online IDE
project_name = test.id
project_dir = [join(EXPORT_WORKSPACE, project_name)]
project_temp = EXPORT_TMP
setup_user_prj(project_dir[0], test.source_dir, test.dependencies)
zip = src is [] # create zip when no src_dir provided
clean = src is [] # don't clean when source is provided, use acrual source tree for IDE files

# Export to selected toolchain
lib_symbols = get_lib_symbols(options.macros, src, p)
tmp_path, report = export(project_dir, project_name, ide, mcu, project_dir[0], project_temp, clean=clean, make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative)
if report['success']:
if not zip:
Expand All @@ -191,12 +153,4 @@
failures.append("%s::%s\t%s"% (mcu, ide, report['errormsg']))

# Prints export results
print
if len(successes) > 0:
print "Successful exports:"
for success in successes:
print " * %s"% success
if len(failures) > 0:
print "Failed exports:"
for failure in failures:
print " * %s"% failure
print_results(successes, failures)
111 changes: 111 additions & 0 deletions tools/project_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
import sys
from os.path import join, abspath, dirname, exists, basename
ROOT = abspath(join(dirname(__file__), ".."))
sys.path.insert(0, ROOT)

from tools.paths import EXPORT_WORKSPACE, EXPORT_TMP
from tools.paths import MBED_BASE, MBED_LIBRARIES
from tools.export import export, setup_user_prj
from tools.utils import mkdir
from tools.tests import Test, TEST_MAP, TESTS
from tools.libraries import LIBRARIES

try:
import tools.private_settings as ps
except:
ps = object()


def get_program(n):
p = TEST_MAP[n].n
return p


def get_test(p):
return Test(p)


def get_test_from_name(n):
if not n in TEST_MAP.keys():
# Check if there is an alias for this in private_settings.py
if getattr(ps, "test_alias", None) is not None:
alias = ps.test_alias.get(n, "")
if not alias in TEST_MAP.keys():
return None
else:
n = alias
else:
return None
return get_program(n)


def get_lib_symbols(macros, src, program):
# Some libraries have extra macros (called by exporter symbols) to we need to pass
# them to maintain compilation macros integrity between compiled library and
# header files we might use with it
lib_symbols = []
if macros:
lib_symbols += macros
if src:
return lib_symbols
test = get_test(program)
for lib in LIBRARIES:
if lib['build_dir'] in test.dependencies:
lib_macros = lib.get('macros', None)
if lib_macros is not None:
lib_symbols.extend(lib_macros)


def setup_project(mcu, ide, program=None, source_dir=None, build=None):

# Some libraries have extra macros (called by exporter symbols) to we need to pass
# them to maintain compilation macros integrity between compiled library and
# header files we might use with it
if source_dir:
# --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
project_dir = source_dir
project_name = TESTS[program] if program else "Unnamed_Project"
project_temp = join(source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu))
mkdir(project_temp)
else:
test = get_test(program)
if not build:
# Substitute the library builds with the sources
# TODO: Substitute also the other library build paths
if MBED_LIBRARIES in test.dependencies:
test.dependencies.remove(MBED_LIBRARIES)
test.dependencies.append(MBED_BASE)

# Build the project with the same directory structure of the mbed online IDE
project_name = test.id
project_dir = [join(EXPORT_WORKSPACE, project_name)]
project_temp = EXPORT_TMP
setup_user_prj(project_dir[0], test.source_dir, test.dependencies)

return project_dir, project_name, project_temp


def perform_export(dir, name, ide, mcu, temp, clean=False, zip=False, lib_symbols='',
sources_relative=False, progen_build=False):

tmp_path, report = export(dir, name, ide, mcu, dir[0], temp, clean=clean,
make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative,
progen_build=progen_build)
return tmp_path, report


def print_results(successes, failures, skips = []):
print
if len(successes) > 0:
print "Successful: "
for success in successes:
print " * %s" % success
if len(failures) > 0:
print "Failed: "
for failure in failures:
print " * %s" % failure
if len(skips) > 0:
print "Skipped: "
for skip in skips:
print " * %s" % skip

Loading

0 comments on commit 4e1ac4d

Please sign in to comment.