Commit 27055d4b authored by Rob Cameron's avatar Rob Cameron
Browse files

Merge branch 'master' into beta-csv

parents b8569c8c fe481746
stages:
- triggers
workflow:
rules:
# If `$FORCE_GITLAB_CI` is set, create a pipeline.
- if: '$FORCE_GITLAB_CI'
# For merge requests, create a pipeline.
- if: '$CI_MERGE_REQUEST_IID'
# For tags, create a pipeline.
- if: '$CI_COMMIT_TAG'
# If commit title starts with 'run-ci:'
- if: '$CI_COMMIT_TITLE =~ /^(r|R)un-ci:/'
trigger_llvm5:
stage: triggers
trigger:
include: .gitlab-pipelines/llvm5/config.yml
strategy: depend
trigger_llvm6:
stage: triggers
trigger:
include: .gitlab-pipelines/llvm6/config.yml
strategy: depend
trigger_llvm7:
stage: triggers
trigger:
include: .gitlab-pipelines/llvm7/config.yml
strategy: depend
trigger_llvm8:
stage: triggers
trigger:
include: .gitlab-pipelines/llvm8/config.yml
strategy: depend
trigger_llvm9:
stage: triggers
trigger:
include: .gitlab-pipelines/llvm9/config.yml
strategy: depend
FROM ponylang/ponyc-ci:llvm-3.9.1
USER root
RUN apt-get update && apt-get -y install software-properties-common
RUN add-apt-repository ppa:mhier/libboost-latest
RUN apt-get update && \
apt-get -y install cmake protobuf-compiler libblkid-dev \
e2fslibs-dev libaudit-dev libboost1.68-dev
stages:
- build
- test
build_llvm39:
stage: build
image: luizperes/parabix:llvm-3.9.1
script:
- mkdir build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4
.test-default: &test-default
stage: test
image: luizperes/parabix:llvm-3.9.1
script:
- mkdir -p build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
test_llvm39:
<<: *test-default
merge_request_llvm39:
<<: *test-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
.test-avx512-default: &test-avx512-default
stage: test
script:
- mkdir -p build && cd build
# Make sure the runner has LLVM installed on /opt/llvm/libllvm{LLVM_VERSION}
- cmake -DCMAKE_PREFIX_PATH=/opt/llvm/libllvm39 -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
tags:
- avx512
test_llvm39_avx512:
<<: *test-avx512-default
merge_request_llvm39_avx512:
<<: *test-avx512-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
FROM ponylang/ponyc-ci:llvm-5.0.2
USER root
RUN apt-get update && apt-get -y install software-properties-common
RUN add-apt-repository ppa:mhier/libboost-latest
RUN apt-get update && \
apt-get -y install cmake protobuf-compiler libblkid-dev \
e2fslibs-dev libaudit-dev libboost1.68-dev
stages:
- build
- test
build_llvm5:
stage: build
image: luizperes/parabix:llvm-5.0.2
script:
- mkdir build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4
.test-default: &test-default
stage: test
image: luizperes/parabix:llvm-5.0.2
script:
- mkdir -p build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
test_llvm5:
<<: *test-default
merge_request_llvm5:
<<: *test-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
.test-avx512-default: &test-avx512-default
stage: test
script:
- mkdir -p build && cd build
# Make sure the runner has LLVM installed on /opt/llvm/libllvm{LLVM_VERSION}
- cmake -DCMAKE_PREFIX_PATH=/opt/llvm/libllvm5 -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
tags:
- avx512
test_llvm5_avx512:
<<: *test-avx512-default
merge_request_avx512_llvm5:
<<: *test-avx512-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
FROM ponylang/ponyc-ci:llvm-6.0.1
USER root
RUN apt-get update && apt-get -y install software-properties-common
RUN add-apt-repository ppa:mhier/libboost-latest
RUN apt-get update && \
apt-get -y install cmake protobuf-compiler libblkid-dev \
e2fslibs-dev libaudit-dev libboost1.68-dev
stages:
- build
- test
build_llvm6:
stage: build
image: luizperes/parabix:llvm-6.0.1
script:
- mkdir build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4
.test-default: &test-default
stage: test
image: luizperes/parabix:llvm-6.0.1
script:
- mkdir -p build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
test_llvm6:
<<: *test-default
merge_request_llvm6:
<<: *test-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
.test-avx-default: &test-avx-default
stage: test
script:
- mkdir -p build && cd build
# Make sure the runner has LLVM installed on /opt/llvm/libllvm{LLVM_VERSION}
# LLVM 6 is currently the default LLVM on cs-osl-08
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
test_avx2_llvm6:
<<: *test-avx-default
tags:
- avx2
merge_request_avx2_llvm6:
<<: *test-avx-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
tags:
- avx2
test_avx512_llvm6:
<<: *test-avx-default
tags:
- avx512
merge_request_avx512_llvm6:
<<: *test-avx-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
tags:
- avx512
FROM ponylang/ponyc-ci:llvm-7.0.1
USER root
RUN apt-get update && apt-get -y install software-properties-common
RUN add-apt-repository ppa:mhier/libboost-latest
RUN apt-get update && \
apt-get -y install cmake protobuf-compiler libblkid-dev \
e2fslibs-dev libaudit-dev libboost1.68-dev
stages:
- build
- test
build_llvm7:
stage: build
image: luizperes/parabix:llvm-7.0.1
script:
- mkdir build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4
.test-default: &test-default
stage: test
image: luizperes/parabix:llvm-7.0.1
script:
- mkdir -p build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
test_llvm7:
<<: *test-default
merge_request_llvm7:
<<: *test-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
.test-avx512-default: &test-avx512-default
stage: test
script:
- mkdir -p build && cd build
# Make sure the runner has LLVM installed on /opt/llvm/libllvm{LLVM_VERSION}
- cmake -DCMAKE_PREFIX_PATH=/opt/llvm/libllvm7 -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
tags:
- avx512
test_avx512_llvm7:
<<: *test-avx512-default
merge_request_avx512_llvm7:
<<: *test-avx512-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
FROM malteschmitz/llvm-8:latest
USER root
RUN apt-get update && apt-get -y install software-properties-common
RUN add-apt-repository ppa:mhier/libboost-latest
RUN apt-get update && \
apt-get -y install protobuf-compiler libblkid-dev \
e2fslibs-dev libaudit-dev libboost1.68-dev
WORKDIR /home
stages:
- build
- test
build_llvm8:
stage: build
image: luizperes/parabix:llvm-8
script:
- mkdir build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4
.test-default: &test-default
stage: test
image: luizperes/parabix:llvm-8
script:
- mkdir -p build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
test_llvm8:
<<: *test-default
merge_request_llvm8:
<<: *test-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
.test-avx512-default: &test-avx512-default
stage: test
script:
- mkdir -p build && cd build
# Make sure the runner has LLVM installed on /opt/llvm/libllvm{LLVM_VERSION}
- cmake -DCMAKE_PREFIX_PATH=/opt/llvm/libllvm8 -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
tags:
- avx512
test_avx512_llvm8:
<<: *test-avx512-default
merge_request_avx512_llvm8:
<<: *test-avx512-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
FROM silkeh/clang:9
USER root
RUN apt-get update && \
apt-get -y install build-essential protobuf-compiler libblkid-dev \
e2fslibs-dev libaudit-dev libncurses5-dev python
WORKDIR /home
RUN wget https://dl.bintray.com/boostorg/release/1.68.0/source/boost_1_68_0.tar.gz \
&& tar -zxvf boost_1_68_0.tar.gz \
&& cd boost_1_68_0/ && ./bootstrap.sh && ./b2
stages:
- build
- test
build_llvm9:
stage: build
image: luizperes/parabix:llvm-9
script:
- mkdir build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release -DBOOST_INCLUDEDIR=/home/boost_1_68_0 ..
- make -j4
.test-default: &test-default
stage: test
image: luizperes/parabix:llvm-9
before_script:
- apt-get -y update && apt-get install -y python python3
script:
- mkdir -p build && cd build
- cmake -DCMAKE_BUILD_TYPE=Release -DBOOST_INCLUDEDIR=/home/boost_1_68_0 ..
- make -j4 check && make kernel-tests
test_llvm9:
<<: *test-default
merge_request_llvm9:
<<: *test-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
.test-avx512-default: &test-avx512-default
stage: test
script:
- mkdir -p build && cd build
# Make sure the runner has LLVM installed on /opt/llvm/libllvm{LLVM_VERSION}
- cmake -DCMAKE_PREFIX_PATH=/opt/llvm/libllvm9 -DCMAKE_BUILD_TYPE=Release ..
- make -j4 check && make kernel-tests
tags:
- avx512
test_avx512_llvm9:
<<: *test-avx512-default
merge_request_avx512_llvm9:
<<: *test-avx512-default
rules:
# If there is a merge request
- if: '$CI_MERGE_REQUEST_IID'
......@@ -20,10 +20,9 @@ enable_testing()
### Build Options ###
option(ENABLE_MULTIPLEXING "Compiling the Multiplexing Module")
option(DISABLE_DUAL_ABI "Disable GCC Dual ABI support" OFF)
option(CARRYPACK_MANAGER "Use CarryPack Manager to reduce space required for carries. For testing only." OFF)
option(USE_ADDRESS_SANITIZER "Enables use of address sanitizer in debug mode if available" OFF)
option(LLVM_PATH "LLVM root directory")
# Build Artifact Configuration
set(ARTIFACT_MODE "Multiple" CACHE STRING "Build framework into a single artifact (Single) or multiple artifacts (Multiple)")
......@@ -55,10 +54,12 @@ add_version_info_from_vcs(Parabix_REVISION)
message(STATUS "Parabix Revision: ${Parabix_REVISION}")
### Import Dependency: LLVM ###
if (LLVM_PATH)
find_package(LLVM REQUIRED CONFIG PATHS ${LLVM_PATH} NO_DEFAULT_PATH)
else()
find_package(LLVM REQUIRED CONFIG)
endif()
message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION} in: ${LLVM_DIR}")
set(LLVM_ALL_TARGETS X86)
llvm_map_components_to_libnames(REQ_LLVM_LIBRARIES ${LLVM_ALL_TARGETS} mcjit native IRReader Linker)
......@@ -78,29 +79,30 @@ set(Boost_USE_STATIC_LIBS ${USE_STATIC_LIBS})
set(Boost_USE_MULTITHREADED OFF)
set(Boost_USE_STATIC_RUNTIME OFF)
find_package(Boost 1.61 REQUIRED COMPONENTS system filesystem iostreams)
message(STATUS "Found Boost_LIBRARY_DIR: ${Boost_LIBRARY_DIR}")
message(STATUS "Found Boost_LIBRARY_DIRS: ${Boost_LIBRARY_DIRS}")
include_directories("${Boost_INCLUDE_DIRS}")
link_directories(${Boost_LIBRARY_DIR})
link_directories(${Boost_LIBRARY_DIRS})
### Import Dependency: Z3 ###
if(ENABLE_MULTIPLEXING)
message(STATUS "Enabling Multiplexing")
find_package(Z3 REQUIRED)
include_directories(${Z3_INCLUDE_DIRS})
# target_link_libraries(PabloADT ${Z3_LIBRARIES})
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_MULTIPLEXING")
endif()
find_package(Z3 REQUIRED) # libz3-dev
message(STATUS "Found Z3 ${Z3_VERSION_STRING} in: ${Z3_INCLUDE_DIR}")
include_directories(${Z3_INCLUDE_DIRS})
string(REGEX REPLACE "([0-9]+)[.].*" "\\1" Z3_VERSION_MAJOR "${Z3_VERSION_STRING}")
string(REGEX REPLACE "[0-9]+[.]([0-9]+).*" "\\1" Z3_VERSION_MINOR "${Z3_VERSION_STRING}")
string(REGEX REPLACE "[0-9]+[.][0-9]+[.]([0-9]+).*" "\\1" Z3_VERSION_PATCH "${Z3_VERSION_STRING}")
math(EXPR Z3_VERSION_INTEGER "${Z3_VERSION_MAJOR} * 10000 + ${Z3_VERSION_MINOR} * 100 + ${Z3_VERSION_PATCH}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DZ3_VERSION_INTEGER=${Z3_VERSION_INTEGER}")
### Project Configuration ###
# Use C++ 11, fail if unable to do so.
# Require that compilers support all features of C11 and C++11.
set(CMAKE_C_STANDARD_REQUIRED ON)
set(CMAKE_C_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
# Define `CXX_STANDARD` to enable conditional compilation based on standard version.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCXX_STANDARD=${CMAKE_CXX_STANDARD}")
......@@ -113,9 +115,6 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
# Parabix Object Cache default
set(PARABIX_OBJECT_CACHE "$ENV{HOME}/.cache/parabix/")
if (NOT EXISTS(${PARABIX_OBJECT_CACHE}))
file(MAKE_DIRECTORY ${PARABIX_OBJECT_CACHE})
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPARABIX_OBJECT_CACHE='\"${PARABIX_OBJECT_CACHE}\"'")
# Use @rpath for dylibs on macOS
......@@ -187,45 +186,35 @@ endfunction(parabix_set_library_name)
include(CMakeParseArguments)
if (CMAKE_BUILD_TYPE STREQUAL "Debug" AND ARTIFACT_TYPE STREQUAL "Static")
set(CMAKE_POSITION_INDEPENDENT_CODE OFF)
# Add assertion debug symbol support checks
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
include(CheckIncludeFileCXX)
CHECK_INCLUDE_FILE_CXX(mach/vm_types.h HAS_MACH_VM_TYPES)
IF (HAS_MACH_VM_TYPES)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_MACH_VM_TYPES")
ELSE()
UNSET(LIBUNWIND_FOUND CACHE)
find_package(Libunwind)
IF (LIBUNWIND_FOUND)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_LIBUNWIND")
include_directories(${LIBUNWIND_INCLUDE_DIR})
target_link_libraries(CodeGen ${LIBUNWIND_LIBRARIES})
ELSE()
CHECK_INCLUDE_FILE_CXX(execinfo.h HAS_EXECINFO)
IF (HAS_EXECINFO)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAS_EXECINFO")
ENDIF()
ENDIF()
ENDIF()
CHECK_INCLUDE_FILE_CXX(execinfo.h HAS_EXECINFO)
if (HAS_EXECINFO)
### Import Dependency: DWARF ###
#find_package(LibDwarf) # libelf-dev libdwarf-dev
#if (LIBDWARF_FOUND)
# include_directories(${LIBDWARF_INCLUDE_DIRS})
# link_directories(${LIBDWARF_LIBRARIES})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_ASSERTION_TRACE")
#endif()
endif()
endif()
# Release & Debug flags
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE} -O3 -DNDEBUG")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG} -O1 -ggdb -fvisibility=default -fno-omit-frame-pointer -fno-optimize-sibling-calls")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE}")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_DEBUG} -O1 -ggdb -fvisibility=default -fno-omit-frame-pointer -fno-optimize-sibling-calls")
function(parabix_add_module)
cmake_parse_arguments(
ARG # resultant argument prefix
"" # boolean args
"NAME" # mono-valued arguments:
# NAME: target name
"SRC;DEPS" # multi-valued arguments:
# SRC: source file list
# DEPS: target dependencies
${ARGN} # arguments to parse
ARG # resultant argument prefix
"USES_Z3" # boolean args
"NAME" # mono-valued arguments:
# NAME: target name
"SRC;DEPS" # multi-valued arguments:
# SRC: source file list
# DEPS: target dependencies
${ARGN} # arguments to parse
)
if(NOT ARG_NAME)
message(FATAL_ERROR "parabix_add_module: A target name is required")
......@@ -242,6 +231,9 @@ function(parabix_add_module)
endif()
parabix_set_library_name(${ARG_NAME})
target_link_libraries(${ARG_NAME} ${ARG_DEPS})
if (ARG_USES_Z3)
target_link_libraries(${ARG_NAME} ${Z3_LIBRARIES})
endif()
endif()
endfunction(parabix_add_module)
......@@ -265,6 +257,9 @@ function(parabix_add_executable)
else()
target_link_libraries(${ARG_NAME} ${ARG_DEPS})
endif()
if(ARTIFACT_MODE STREQUAL "Single")
target_link_libraries(${ARG_NAME} ${Z3_LIBRARIES})
endif()
endfunction(parabix_add_executable)
# Creates a pablo source target which copies pablo source files to the parabix
......@@ -276,7 +271,7 @@ function(parabix_add_pablo_src NAME)
foreach(PABLO_FILE ${ARGN})
list(APPEND FILE_LIST "${CMAKE_CURRENT_SOURCE_DIR}/${PABLO_FILE}")
endforeach(PABLO_FILE)
add_custom_target(${NAME}
COMMAND cp ${FILE_LIST} ${PARABIX_OBJECT_CACHE}
DEPENDS ${ARGN})
......
......@@ -24,8 +24,7 @@ import sys
import codecs
import random
sys.stdout = codecs.getwriter('utf8')(sys.stdout)
sys.stderr = codecs.getwriter('utf8')(sys.stderr)
in_datafile = False
dataFileName = ""
......@@ -160,11 +159,12 @@ flag_map = {'-CarryMode' : ['Compressed', 'BitBlock'],
def add_random_flags(flags, fileLength):
selected = {}
flag_keys = list(flag_map.keys())
for i in range(options.random_flag_count):
rand_flag = flag_map.keys()[random.randint(0, len(flag_map) - 1)]
rand_flag = flag_keys[random.randint(0, len(flag_map) - 1)]
# Avoid duplicate flags and expensive test cases
while rand_flag in selected or (rand_flag == "-v" and fileLength > 4000):
rand_flag = flag_map.keys()[random.randint(0, len(flag_map) - 1)]
rand_flag = flag_keys[random.randint(0, len(flag_map) - 1)]
selected[rand_flag] = True
values = flag_map[rand_flag]
if values == []:
......@@ -258,6 +258,7 @@ if __name__ == '__main__':
sys.exit(1)
random.seed(options.random_seed)
grep_program_under_test = args[0]
print("grep_program: %s" % grep_program_under_test)
grep_test_file = open(os.path.join(QA_dir,options.testcases), 'r')
grep_test_spec = grep_test_file.read()
grep_test_file.close()
......
# icgrep
> This is the open-source version of `icgrep 1.x`
icgrep is a very fast regular expression search program, particularly for complex regular expressions. It is also a very capable engine, supporting most common regular expression syntax and many useful command line options.
### Usage
`icgrep 1.0` is designed to offer substantial Unicode support, meeting **all** the Unicode Level 1 requirements of UTS #18, the Unicode Technical Standard for regular expressions. Development of `icgrep 2.0` is on track to meet the Unicode level 2 requirements of UTS #18.
##### Regex matching
Normal usage to find lines in a file `f` matching a regular expression `r` is:
`icgrep r f`
##### Counting matching lines
To produce a count of matching lines only, use the flag `-c` such as below:
`icgrep -c r f`
To read the regular expression to be matched from file `regexpf` use the flag `-f` such as below:
`icgrep -f regexpf f`
### Build
`icgrep` is one of the tools available on `Parabix`. Check the [README.md](README.md) file for more information.
### License
LLVM files are governed by the LLVM Release License in [LLVM-LICENSE.txt](LLVM-LICENSE.txt). `icgrep` is governed by Open Software License 3.0 in [OSL-3.0.txt](OSL-3.0.txt).
README-icgrep.txt
This is the open-source version of icgrep 1.x.
icgrep is a very fast regular expression search program, particularly
for complex regular expressions. It is also a very capable engine,
supporting most common regular expression syntax and many useful
command line options.
icgrep 1.0 is designed to offer substantial Unicode support, meeting
all the Unicode Level 1 requirements of UTS #18, the Unicode
Technical Standard for regular expressions. Development of icgrep 2.0 is
on track to meet the Unicode level 2 requirements of UTS #18.
Normal usage to find lines in a file f matching a regexp r is:
icgrep r f
To produce a count of matching lines only, use the command:
icgrep -c r f
To read the regexp to be matched from file regexpf use the command:
icgrep -f regexpf f
BUILD
To build icgrep, you need a development environment that meets
several requirements.
- A modern C++ compiler supporting C++ 11.
- The cmake build system version 2.8 or better.
- Boost libraries version 1.61 or better.
- Standard C++ development tools including git, C++, etc.
- An LLVM system version 3.9 or better.
Clone parabix-devel from the repository:
git clone https://cs-git-research.cs.surrey.sfu.ca/cameron/parabix-devel.git
Create a build subdirectory
cd parabix-devel
mkdir build
cd build
Create the makefiles
cmake -DCMAKE_BUILD_TYPE=Release ..
Note: if you have built/installed a custom LLVM version, you may need
to override cmake's default search path.
cmake -DCMAKE_PREFIX_PATH=path/to/libllvm -DCMAKE_BUILD_TYPE=Release ..
Then, while you are still in the build directory, build the software and
run the test suite, by issuing the following two commands.
make
make check
LLVM files are governed by the LLVM Release License in LLVM-LICENSE.txt.
icgrep is governed by Open Software License 3.0 in OSL-3.0.txt.
# Parabix
> Parabix technology is a high-performance programming framework for streaming text processing applications, leveraging both SIMD and multicore parallel processing features.
[![pipeline status](https://cs-git-research.cs.surrey.sfu.ca/cameron/parabix-devel/badges/master/pipeline.svg)](https://cs-git-research.cs.surrey.sfu.ca/cameron/parabix-devel/-/commits/master) [![Gitter](https://badges.gitter.im/parabix-devel/community.svg)](https://gitter.im/parabix-devel/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
### Documentation
For more information about Parabix, please check our [wiki](https://cs-git-research.cs.surrey.sfu.ca/cameron/parabix-devel/-/wikis/home) or reach out to us on [Gitter](https://gitter.im/parabix-devel/community).
### Tools
Parabix includes a set of built-in [tools](https://cs-git-research.cs.surrey.sfu.ca/cameron/parabix-devel/-/tree/master/tools), such as [`icgrep`](README-icgrep.md), which is a very fast regular expression search program.
### Requirements
To build Parabix, you need a development environment that meets a few requirements.
- Standard C++ development tools including git, C++, etc.
- A modern C++ compiler supporting at least C++ 11.
- The [`cmake`](https://cmake.org/download/) build system version 2.8 or better.
- [`Boost`](https://www.boost.org/users/download/) libraries version `1.61` or better.
- [`Z3`](https://github.com/Z3Prover/z3) Theorem Prover.
- An [`LLVM`](https://releases.llvm.org/download.html) system version `5` or better.
### Build
- Clone parabix-devel from the repository:
`git clone https://cs-git-research.cs.surrey.sfu.ca/cameron/parabix-devel.git`
- Create a build subdirectory
- `cd parabix-devel`
- `mkdir build`
- `cd build`
- Create the makefiles
`cmake -DCMAKE_BUILD_TYPE=Release ..`
- _Note: if you have built/installed a custom LLVM version, you may need to override cmake's default search path._
`cmake -DCMAKE_PREFIX_PATH=path/to/libllvm -DCMAKE_BUILD_TYPE=Release ..`
- `make`
- If you wish to run the test suite, while you are still in the build directory, run the following command:
`make check`
In the configuration above, all compiled tools will then be found on `/path/to/parabix-devel/build/bin`
### Papers
[Bitwise Data Parallelism with LLVM: The ICgrep Case Study](https://link.springer.com/chapter/10.1007%2F978-3-319-27122-4_26). Robert D. Cameron, Nigel Medforth, Dan Lin, Dale Denis, William N. Sumner. ICA3PP (2) 2015: 373-387
[Bitwise data parallelism in regular expression matching](https://dl.acm.org/doi/10.1145/2628071.2628079). Robert D. Cameron, Thomas C. Shermer, Arrvindh Shriraman, Kenneth S. Herdy, Dan Lin, Benjamin R. Hull, Meng Lin. PACT 2014: 139-150
[icXML: Accelerating a Commercial XML Parser Using SIMD and Multicore Technologies](http://www.balisage.net/Proceedings/vol10/html/Cameron01/BalisageVol10-Cameron01.html). Presented at Balisage: The Markup Conference 2013, Montréal, Canada, August 6 - 9, 2013.
[Parabix: Boosting the Efficiency of Text Processing on Commodity Processors](http://www.cs.sfu.ca/~ashriram/publications/2012_HPCA_Parabix.pdf), 18th International Symposium on High Performance Computer Architecture in New Orleans, Louisiana, February 2012
[Parallel Scanning with Bitstream Addition: An XML Case Study](http://parabix.costar.sfu.ca/export/901/docs/EuroPar2011/europar-cameron.pdf), Euro-Par 2011 Conference, Bordeaux, France, September 2011.
[Parallel Bit Stream Technology as a Foundation for XML Parsing Performance](http://www.balisage.net/Proceedings/vol4/html/Cameron01/BalisageVol4-Cameron01.html), 2009 International Symposium on Processing XML Efficiently: Overcoming Limits on Space, Time, or Bandwidth, Montreal, Canada, August 2009.
[Architectural Support for SIMD Text Processing with Parallel Bit Streams: The Inductive Doubling Principle](http://parabix.costar.sfu.ca/export/901/docs/ASPLOS09/asplos094-cameron.pdf), Fourteenth International Conference on Architectural Support for Programming Languages and Operating Systems (ASPLOS '09), Washington, DC, March 2009.
### License
LLVM files are governed by the LLVM Release License in [LLVM-LICENSE.txt](LLVM-LICENSE.txt). `Parabix` is governed by Open Software License `3.0` in [OSL3.0.txt](OSL3.0.txt).
IF (Z3_INCLUDE_DIR)
SET(Z3_FIND_QUIETLY TRUE)
ENDIF (Z3_INCLUDE_DIR)
INCLUDE(CheckCXXSourceRuns)
FIND_PATH(Z3_INCLUDE_DIR z3.h z3++.h)
# Function to check Z3's version
function(check_z3_version z3_include z3_lib)
# The program that will be executed to print Z3's version.
file(WRITE ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testz3.c
"#include <assert.h>
#include <z3.h>
int main() {
unsigned int major, minor, build, rev;
Z3_get_version(&major, &minor, &build, &rev);
printf(\"%u.%u.%u\", major, minor, build);
return 0;
}")
SET(Z3_NAMES z3 libz3 ltz3 libz3 lz3)
FIND_LIBRARY(Z3_LIBRARY NAMES ${Z3_NAMES} )
# Get lib path
get_filename_component(z3_lib_path ${z3_lib} PATH)
INCLUDE(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(Z3 DEFAULT_MSG Z3_LIBRARY Z3_INCLUDE_DIR)
try_run(
Z3_RETURNCODE
Z3_COMPILED
${CMAKE_BINARY_DIR}
${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testz3.c
COMPILE_DEFINITIONS -I"${z3_include}"
LINK_LIBRARIES -L${z3_lib_path} -lz3
RUN_OUTPUT_VARIABLE SRC_OUTPUT
)
IF(Z3_FOUND)
SET(Z3_LIBRARIES ${Z3_LIBRARY})
SET(Z3_INCLUDE_DIRS ${Z3_INCLUDE_DIR})
ELSE(Z3_FOUND)
SET(Z3_LIBRARIES)
ENDIF(Z3_FOUND)
if(Z3_COMPILED)
string(REGEX REPLACE "([0-9]*\\.[0-9]*\\.[0-9]*\\.[0-9]*)" "\\1"
z3_version "${SRC_OUTPUT}")
set(Z3_VERSION_STRING ${z3_version} PARENT_SCOPE)
endif()
endfunction(check_z3_version)
MARK_AS_ADVANCED(Z3_LIBRARY Z3_INCLUDE_DIR)
# Looking for Z3 in LLVM_Z3_INSTALL_DIR
find_path(Z3_INCLUDE_DIR NAMES z3.h
NO_DEFAULT_PATH
PATHS ${LLVM_Z3_INSTALL_DIR}/include
PATH_SUFFIXES libz3 z3
)
find_library(Z3_LIBRARIES NAMES z3 libz3
NO_DEFAULT_PATH
PATHS ${LLVM_Z3_INSTALL_DIR}
PATH_SUFFIXES lib bin
)
# If Z3 has not been found in LLVM_Z3_INSTALL_DIR look in the default directories
find_path(Z3_INCLUDE_DIR NAMES z3.h
PATH_SUFFIXES libz3 z3
)
find_library(Z3_LIBRARIES NAMES z3 libz3
PATH_SUFFIXES lib bin
)
# Searching for the version of the Z3 library is a best-effort task
unset(Z3_VERSION_STRING)
# First, try to check it dynamically, by compiling a small program that
# prints Z3's version
if(Z3_INCLUDE_DIR AND Z3_LIBRARIES)
# We do not have the Z3 binary to query for a version. Try to use
# a small C++ program to detect it via the Z3_get_version() API call.
check_z3_version(${Z3_INCLUDE_DIR} ${Z3_LIBRARIES})
endif()
# If the dynamic check fails, we might be cross compiling: if that's the case,
# check the version in the headers, otherwise, fail with a message
if(NOT Z3_VERSION_STRING AND (CMAKE_CROSSCOMPILING AND
Z3_INCLUDE_DIR AND
EXISTS "${Z3_INCLUDE_DIR}/z3_version.h"))
# TODO: print message warning that we couldn't find a compatible lib?
# Z3 4.8.1+ has the version is in a public header.
file(STRINGS "${Z3_INCLUDE_DIR}/z3_version.h"
z3_version_str REGEX "^#define[\t ]+Z3_MAJOR_VERSION[\t ]+.*")
string(REGEX REPLACE "^.*Z3_MAJOR_VERSION[\t ]+([0-9]).*$" "\\1"
Z3_MAJOR "${z3_version_str}")
file(STRINGS "${Z3_INCLUDE_DIR}/z3_version.h"
z3_version_str REGEX "^#define[\t ]+Z3_MINOR_VERSION[\t ]+.*")
string(REGEX REPLACE "^.*Z3_MINOR_VERSION[\t ]+([0-9]).*$" "\\1"
Z3_MINOR "${z3_version_str}")
file(STRINGS "${Z3_INCLUDE_DIR}/z3_version.h"
z3_version_str REGEX "^#define[\t ]+Z3_BUILD_NUMBER[\t ]+.*")
string(REGEX REPLACE "^.*Z3_BUILD_VERSION[\t ]+([0-9]).*$" "\\1"
Z3_BUILD "${z3_version_str}")
set(Z3_VERSION_STRING ${Z3_MAJOR}.${Z3_MINOR}.${Z3_BUILD})
unset(z3_version_str)
endif()
if(NOT Z3_VERSION_STRING)
# Give up: we are unable to obtain a version of the Z3 library. Be
# conservative and force the found version to 0.0.0 to make version
# checks always fail.
set(Z3_VERSION_STRING "0.0.0")
endif()
# handle the QUIETLY and REQUIRED arguments and set Z3_FOUND to TRUE if
# all listed variables are TRUE
include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(Z3
REQUIRED_VARS Z3_LIBRARIES Z3_INCLUDE_DIR
VERSION_VAR Z3_VERSION_STRING)
mark_as_advanced(Z3_INCLUDE_DIR Z3_LIBRARIES)
......@@ -15,8 +15,12 @@
#include <llvm/IR/Function.h>
#endif
#include <unistd.h>
#ifdef ENABLE_ASSERTION_TRACE
#include <llvm/ADT/DenseMap.h>
#endif
#include <util/not_null.h>
namespace kernels { class KernelBuilder; }
namespace kernel { class KernelBuilder; }
namespace llvm { class Function; }
namespace llvm { class IntegerType; }
namespace llvm { class Module; }
......@@ -52,24 +56,27 @@ public:
void setModule(llvm::Module * module) {
mModule = module;
ClearInsertionPoint();
#ifdef ENABLE_ASSERTION_TRACE
mBacktraceSymbols.clear();
#endif
}
// UDiv and URem with optimization for division by power-of-2 constants
llvm::Value * CreateUDiv(llvm::Value * number, llvm::Value * divisor, const llvm::Twine &Name = "");
llvm::Value * CreateURem(llvm::Value * number, llvm::Value * divisor, const llvm::Twine &Name = "");
llvm::Value * CreateUDiv(llvm::Value * number, llvm::Value * divisor, const llvm::Twine Name = "");
llvm::Value * CreateURem(llvm::Value * number, llvm::Value * divisor, const llvm::Twine Name = "");
// Division with rounding up to the ceiling
// Equivalent to CreateUDiv(CreateAdd(number, CreateSub(divisor, ConstantInt::get(divisor->getType(), 1))), divisor)
llvm::Value * CreateCeilUDiv(llvm::Value * number, llvm::Value * divisor, const llvm::Twine &Name = "");
llvm::Value * CreateCeilUDiv(llvm::Value * number, llvm::Value * divisor, const llvm::Twine Name = "");
// Round down to a multiple of divisor.
llvm::Value * CreateRoundDown(llvm::Value * number, llvm::Value * divisor, const llvm::Twine &Name = "");
llvm::Value * CreateRoundDown(llvm::Value * number, llvm::Value * divisor, const llvm::Twine Name = "");
// Round up to a multiple of divisor.
llvm::Value * CreateRoundUp(llvm::Value * number, llvm::Value * divisor, const llvm::Twine &Name = "");
llvm::Value * CreateRoundUp(llvm::Value * number, llvm::Value * divisor, const llvm::Twine Name = "");
// Get minimum of two unsigned numbers
llvm::Value * CreateUMin(llvm::Value * const a, llvm::Value * const b, const llvm::Twine &Name = "") {
llvm::Value * CreateUMin(llvm::Value * const a, llvm::Value * const b, const llvm::Twine Name = "") {
if (LLVM_UNLIKELY(a == nullptr || a == b)) return b;
if (LLVM_UNLIKELY(b == nullptr)) return a;
assert (a->getType() == b->getType());
......@@ -77,7 +84,7 @@ public:
}
// Get minimum of two signed numbers
llvm::Value * CreateSMin(llvm::Value * const a, llvm::Value * const b, const llvm::Twine &Name = "") {
llvm::Value * CreateSMin(llvm::Value * const a, llvm::Value * const b, const llvm::Twine Name = "") {
if (LLVM_UNLIKELY(a == nullptr || a == b)) return b;
if (LLVM_UNLIKELY(b == nullptr)) return a;
assert (a->getType() == b->getType());
......@@ -85,7 +92,7 @@ public:
}
// Get maximum of two unsigned numbers
llvm::Value * CreateUMax(llvm::Value * const a, llvm::Value * const b, const llvm::Twine &Name = "") {
llvm::Value * CreateUMax(llvm::Value * const a, llvm::Value * const b, const llvm::Twine Name = "") {
if (LLVM_UNLIKELY(a == nullptr || a == b)) return b;
if (LLVM_UNLIKELY(b == nullptr)) return a;
assert (a->getType() == b->getType());
......@@ -93,16 +100,16 @@ public:
}
// Get maximum of two signed numbers
llvm::Value * CreateSMax(llvm::Value * const a, llvm::Value * const b, const llvm::Twine &Name = "") {
llvm::Value * CreateSMax(llvm::Value * const a, llvm::Value * const b, const llvm::Twine Name = "") {
if (LLVM_UNLIKELY(a == nullptr || a == b)) return b;
if (LLVM_UNLIKELY(b == nullptr)) return a;
assert (a->getType() == b->getType());
return CreateSelect(CreateICmpSGT(a, b), a, b, Name);
}
llvm::Value * CreateSaturatingAdd(llvm::Value * const a, llvm::Value * const b, const llvm::Twine &Name = "");
llvm::Value * CreateSaturatingAdd(llvm::Value * const a, llvm::Value * const b, const llvm::Twine Name = "");
llvm::Value * CreateSaturatingSub(llvm::Value * const a, llvm::Value * const b, const llvm::Twine &Name = "");
llvm::Value * CreateSaturatingSub(llvm::Value * const a, llvm::Value * const b, const llvm::Twine Name = "");
llvm::Value * CreateMalloc(llvm::Value * const size);
......@@ -265,7 +272,7 @@ public:
void CallPrintIntCond(llvm::StringRef name, llvm::Value * const value, llvm::Value * const cond, const STD_FD fd = STD_FD::STD_ERR);
void CallPrintInt(llvm::StringRef name, llvm::Value * const value, const STD_FD fd = STD_FD::STD_ERR);
llvm::CallInst * CallPrintInt(llvm::StringRef name, llvm::Value * const value, const STD_FD fd = STD_FD::STD_ERR);
llvm::Constant * GetString(llvm::StringRef Str);
......@@ -328,7 +335,7 @@ public:
return CreateLikelyCondBr(Cond, True, False, 100 - probability);
}
llvm::AllocaInst * CreateAllocaAtEntryPoint(llvm::Type * Ty, llvm::Value * ArraySize = nullptr, const llvm::Twine & Name = "");
llvm::AllocaInst * CreateAllocaAtEntryPoint(llvm::Type * Ty, llvm::Value * ArraySize = nullptr, const llvm::Twine Name = "");
llvm::BasicBlock * CreateBasicBlock(const llvm::StringRef name = "", llvm::BasicBlock * insertBefore = nullptr);
......@@ -337,25 +344,25 @@ public:
llvm::Value * CreatePopcount(llvm::Value * bits);
// TODO: AVX512 offers these as vector instructions
llvm::Value * CreateCountForwardZeroes(llvm::Value * value, const bool guaranteedNonZero = false);
llvm::Value * CreateCountReverseZeroes(llvm::Value * value, const bool guaranteedNonZero = false);
llvm::Value * CreateCountForwardZeroes(llvm::Value * value, const llvm::Twine Name = "", const no_conversion<bool> guaranteedNonZero = false);
llvm::Value * CreateCountReverseZeroes(llvm::Value * value, const llvm::Twine Name = "", const no_conversion<bool> guaranteedNonZero = false);
// Useful bit manipulation operations
llvm::Value * CreateResetLowestBit(llvm::Value * bits, const llvm::Twine &Name = "");
llvm::Value * CreateResetLowestBit(llvm::Value * bits, const llvm::Twine Name = "");
llvm::Value * CreateIsolateLowestBit(llvm::Value * bits, const llvm::Twine &Name = "");
llvm::Value * CreateIsolateLowestBit(llvm::Value * bits, const llvm::Twine Name = "");
llvm::Value * CreateMaskToLowestBitInclusive(llvm::Value * bits, const llvm::Twine &Name = "");
llvm::Value * CreateMaskToLowestBitInclusive(llvm::Value * bits, const llvm::Twine Name = "");
llvm::Value * CreateMaskToLowestBitExclusive(llvm::Value * bits, const llvm::Twine &Name = "");
llvm::Value * CreateMaskToLowestBitExclusive(llvm::Value * bits, const llvm::Twine Name = "");
virtual llvm::Value * CreateZeroHiBitsFrom(llvm::Value * bits, llvm::Value * pos, const llvm::Twine &Name = "");
virtual llvm::Value * CreateZeroHiBitsFrom(llvm::Value * bits, llvm::Value * pos, const llvm::Twine Name = "");
virtual llvm::Value * CreateExtractBitField(llvm::Value * bits, llvm::Value * start, llvm::Value * length, const llvm::Twine &Name = "");
virtual llvm::Value * CreateExtractBitField(llvm::Value * bits, llvm::Value * start, llvm::Value * length, const llvm::Twine Name = "");
llvm::Value * CreateCeilLog2(llvm::Value * value, const llvm::Twine &Name = "");
llvm::Value * CreateCeilLog2(llvm::Value * value, const llvm::Twine Name = "");
llvm::Value * CreateLog2(llvm::Value * value, const llvm::Twine &Name = "");
llvm::Value * CreateLog2(llvm::Value * value, const llvm::Twine Name = "");
llvm::Value * CreateReadCycleCounter();
......@@ -373,19 +380,19 @@ public:
virtual llvm::LoadInst * CreateLoad(llvm::Value * Ptr, const char * Name);
virtual llvm::LoadInst * CreateLoad(llvm::Value * Ptr, const llvm::Twine & Name = "");
virtual llvm::LoadInst * CreateLoad(llvm::Value * Ptr, const llvm::Twine Name = "");
virtual llvm::LoadInst * CreateLoad(llvm::Type * Ty, llvm::Value * Ptr, const llvm::Twine & Name = "");
virtual llvm::LoadInst * CreateLoad(llvm::Type * Ty, llvm::Value * Ptr, const llvm::Twine Name = "");
virtual llvm::LoadInst * CreateLoad(llvm::Value * Ptr, bool isVolatile, const llvm::Twine & Name = "");
virtual llvm::LoadInst * CreateLoad(llvm::Value * Ptr, bool isVolatile, const llvm::Twine Name = "");
virtual llvm::StoreInst * CreateStore(llvm::Value * Val, llvm::Value * Ptr, bool isVolatile = false);
llvm::LoadInst * CreateAlignedLoad(llvm::Value * Ptr, unsigned Align, const char * Name);
llvm::LoadInst * CreateAlignedLoad(llvm::Value * Ptr, unsigned Align, const llvm::Twine & Name = "");
llvm::LoadInst * CreateAlignedLoad(llvm::Value * Ptr, unsigned Align, const llvm::Twine Name = "");
llvm::LoadInst * CreateAlignedLoad(llvm::Value * Ptr, unsigned Align, bool isVolatile, const llvm::Twine & Name = "");
llvm::LoadInst * CreateAlignedLoad(llvm::Value * Ptr, unsigned Align, bool isVolatile, const llvm::Twine Name = "");
llvm::StoreInst * CreateAlignedStore(llvm::Value * Val, llvm::Value * Ptr, unsigned Align, bool isVolatile = false);
......@@ -427,15 +434,15 @@ public:
llvm::MDNode *ScopeTag = nullptr,
llvm::MDNode *NoAliasTag = nullptr);
llvm::Value * CreateExtractElement(llvm::Value *Vec, llvm::Value *Idx, const llvm::Twine &Name = "");
llvm::Value * CreateExtractElement(llvm::Value *Vec, llvm::Value *Idx, const llvm::Twine Name = "");
llvm::Value * CreateExtractElement(llvm::Value *Vec, uint64_t Idx, const llvm::Twine &Name = "") {
llvm::Value * CreateExtractElement(llvm::Value *Vec, uint64_t Idx, const llvm::Twine Name = "") {
return CreateExtractElement(Vec, getInt64(Idx), Name);
}
llvm::Value * CreateInsertElement(llvm::Value *Vec, llvm::Value *NewElt, llvm::Value *Idx, const llvm::Twine &Name = "");
llvm::Value * CreateInsertElement(llvm::Value *Vec, llvm::Value *NewElt, llvm::Value *Idx, const llvm::Twine Name = "");
llvm::Value * CreateInsertElement(llvm::Value *Vec, llvm::Value *NewElt, uint64_t Idx, const llvm::Twine &Name = "") {
llvm::Value * CreateInsertElement(llvm::Value *Vec, llvm::Value *NewElt, uint64_t Idx, const llvm::Twine Name = "") {
return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name);
}
......@@ -499,6 +506,9 @@ protected:
codegen::VirtualDriver * mDriver;
llvm::LLVMContext mContext;
const std::string mTriple;
#ifdef ENABLE_ASSERTION_TRACE
llvm::DenseMap<uintptr_t, llvm::Constant *> mBacktraceSymbols;
#endif
};
template <typename ExternalFunctionType>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment