Python

.gitignore

@@ -117,6 +117,7 @@ website/i18n/*
 cmake-build-debug
 .ipynb_checkpoints
 .python-version
+.pytest_cache
 
 package.json.lerna_backup
 website/static/css/material.dark.css

.travis.yml

@@ -17,21 +17,27 @@ matrix:
 
     - python: "3.6"
       language: python
-      env: TEST=PYTHON_TABLE
+      env: TEST=PYTHON
       cache: pip
       services: docker
 
-    - language: c++
-      env: TEST=CPP_OSX
-      os: osx
-
-  allow_failures:
     - python: "3.6"
       language: python
       env: TEST=PYTHON_TABLE
       cache: pip
       services: docker
 
+    - python: "3.6"
+      language: python
+      env: TEST=PYTHON_NODE
+      cache: pip
+      services: docker
+
+    - language: c++
+      env: TEST=CPP_OSX
+      os: osx
+
+  allow_failures:
     - language: c++
       env: TEST=CPP_OSX
       os: osx
@@ -42,16 +48,17 @@ env:
 
 install:
   - if [ "$TEST" = "CPP_OSX" ]; then brew install yarn; fi
+  - if [ "$TEST" = "PYTHON_NODE"]; then curl -sL https://deb.nodesource.com/setup_8.x | bash - && apt-get install npm; fi
   - yarn
- 
+
 script:
   - if [ "$TEST" = "WASM" ]; then PSP_DOCKER=1 yarn lint; fi
   - if [ "$TEST" = "WASM" ]; then PSP_DOCKER=1 yarn build; fi
   - if [ "$TEST" = "WASM" ]; then PSP_DOCKER=1 yarn test --quiet; fi
   - if [ "$TEST" = "CPP" ]; then PSP_DOCKER=1 yarn build_cpp; fi
   - if [ "$TEST" = "CPP" ]; then PSP_DOCKER=1 yarn test_cpp; fi
-  - if [ "$TEST" = "PYTHON_TABLE" ]; then PSP_DOCKER=1 yarn build_python:table; fi
-  - if [ "$TEST" = "PYTHON_TABLE" ]; then PSP_DOCKER=1 yarn lint_python:table; fi
-  - if [ "$TEST" = "PYTHON_TABLE" ]; then PSP_DOCKER=1 yarn test_python:table; fi
+  - if [ "$TEST" = "PYTHON" ]; then PSP_DOCKER=1 yarn ci_python; fi
+  - if [ "$TEST" = "PYTHON_TABLE" ]; then PSP_DOCKER=1 yarn ci_python:table; fi
+  - if [ "$TEST" = "PYTHON_NODE" ]; then PSP_DOCKER=1 yarn ci_python:node; fi
   - if [ "$TEST" = "CPP_OSX" ]; then yarn build_cpp; fi
   - if [ "$TEST" = "CPP_OSX" ]; then yarn test_cpp; fi

cmake/Pybind.txt.in

@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 2.8.2)
+
+project(pybind11-download NONE)
+
+include(ExternalProject)
+ExternalProject_Add(pybind11
+  GIT_REPOSITORY    https://github.com/pybind/pybind11.git
+  GIT_TAG           master
+  SOURCE_DIR        "${CMAKE_BINARY_DIR}/pybind11-src"
+  BINARY_DIR        "${CMAKE_BINARY_DIR}/pybind11-build"
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND     ""
+  INSTALL_COMMAND   ""
+  TEST_COMMAND      ""
+  CMAKE_ARGS        "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+)

cpp/perspective/CMakeLists.txt

@@ -245,14 +245,19 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 		find_package( PythonInterp 3.7 REQUIRED )
 		find_package( PythonLibs 3.7 REQUIRED )
 		include_directories( ${PYTHON_INCLUDE_DIRS} )
-		find_package(NumPy)
 
-		find_package( Boost COMPONENTS python REQUIRED )
-		find_library( BOOST_PYTHON boost_python)
-		find_library( BOOST_NUMPY boost_numpy)
-		include_directories( ${Boost_INCLUDE_DIR} )
+		find_package(pybind11)
+		if(NOT pybind11_FOUND)
+		    message("${Red}PyBind11 could not be located${ColorReset}")
+			psp_build_dep("pybind11" "../../cmake/Pybind.txt.in")
+		else()
+		    message("${Cyan}Found PyBind11 in ${pybind11_INCLUDE_DIRS}${ColorReset}")
+		endif()
+		include_directories( ${pybind11_INCLUDE_DIRS} )
 
+		find_package(NumPy)
 		include_directories( ${PYTHON_NUMPY_INCLUDE_DIR})
+
 		#####################
 	endif()
 endif()
@@ -350,6 +355,17 @@ set (SOURCE_FILES
 	src/cpp/vocab.cpp
 	)
 
+set (PYTHON_SOURCE_FILES
+	${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/accessor.cpp
+	${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/context.cpp
+	${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/fill.cpp
+	${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/python.cpp
+	${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/serialization.cpp
+	${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/table.cpp
+	${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/utils.cpp
+	${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/view.cpp
+	)
+
 if (WIN32)
 	set(CMAKE_CXX_FLAGS " /EHsc")
 else()
@@ -376,22 +392,19 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 		########################
 		# Python extra targets #
 		########################
-		add_library(psp SHARED ${SOURCE_FILES} ${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/numpy.cpp ${HEADER_FILES})
+		add_library(psp SHARED ${SOURCE_FILES} ${HEADER_FILES})
 
 		target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1)
-		target_link_libraries(psp ${BOOST_PYTHON})
-		target_link_libraries(psp ${BOOST_NUMPY})
-		target_link_libraries(psp ${Boost_LIBRARIES} ${PYTHON_LIBRARIES})
+		target_link_libraries(psp ${PYTHON_LIBRARIES})
 		set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
 
-		add_library(binding SHARED ${CMAKE_SOURCE_DIR}/../../python/table/perspective/src/python.cpp)
+		add_library(binding SHARED ${PYTHON_SOURCE_FILES})
 
 		target_compile_definitions(binding PRIVATE PSP_ENABLE_PYTHON=1)
+		target_compile_options(binding PRIVATE -Wdeprecated-declarations)
 		target_link_libraries(binding psp)
 		target_link_libraries(binding tbb)
-		target_link_libraries(binding ${BOOST_PYTHON})
-		target_link_libraries(binding ${BOOST_NUMPY})
-		target_link_libraries(binding ${Boost_LIBRARIES} ${PYTHON_LIBRARIES})
+		target_link_libraries(binding ${PYTHON_LIBRARIES})
 		########################
 	else()
 		add_library(psp SHARED ${SOURCE_FILES} ${HEADER_FILES})

cpp/perspective/src/cpp/column.cpp

@@ -16,12 +16,6 @@ SUPPRESS_WARNINGS_VC(4505)
 #include <perspective/sym_table.h>
 #include <tsl/hopscotch_set.h>
 
-#ifdef PSP_ENABLE_PYTHON
-namespace py = boost::python;
-namespace np = boost::python::numpy;
-#include <perspective/numpy.h>
-#endif
-
 namespace perspective {
 // TODO : move to delegated constructors in C++11
 
@@ -922,7 +916,7 @@ t_column::borrow_vocabulary(const t_column& o) {
 }
 
 #ifdef PSP_ENABLE_PYTHON
-np::ndarray
+py::array
 t_column::_as_numpy() {
     if (is_vlen_dtype(m_dtype))
         return m_data->_as_numpy(DTYPE_UINT64);

cpp/perspective/src/cpp/date.cpp

@@ -68,6 +68,21 @@ t_date::consecutive_day_idx() const {
         + CUMULATIVE_DAYS[leap_selector][m - 1];
 }
 
+std::tm
+t_date::get_tm() const {
+    std::tm rval;
+
+    rval.tm_year = year() - 1900; // tm years are since 1900
+    rval.tm_mon = month() - 1; // tm months from 0
+    rval.tm_mday = day();
+    rval.tm_hour = 0;
+    rval.tm_min = 0;
+    rval.tm_sec = 0;
+    rval.tm_isdst = -1; // let std::mktime decide whether DST is in effect
+
+    return rval;
+}
+
 t_date
 from_consecutive_day_idx(std::int32_t idx) {
     std::int32_t y = static_cast<std::int32_t>(idx / 365.2425);

cpp/perspective/src/cpp/emscripten.cpp

@@ -1391,7 +1391,7 @@ namespace binding {
 
     template <>
     std::tuple<std::string, std::string, std::vector<t_tscalar>>
-    make_filter_term(t_dtype column_type, t_val date_parser, const std::string column_name, const std::string& filter_op_str, t_val filter_term) {
+    make_filter_term(t_dtype column_type, t_val date_parser, const std::string& column_name, const std::string& filter_op_str, t_val filter_term) {
         t_filter_op filter_op = str_to_filter_op(filter_op_str);
         std::vector<t_tscalar> terms;
 

cpp/perspective/src/cpp/pool.cpp

@@ -41,7 +41,7 @@ t_pool::t_pool()
 #else
 
 t_pool::t_pool()
-    : m_sleep(100)
+    : m_sleep(0)
     , m_has_python_dep(false) {
     m_run.clear();
 }

cpp/perspective/src/cpp/storage.cpp

@@ -36,12 +36,6 @@ SUPPRESS_WARNINGS_VC(4505)
 #include <unistd.h>
 #endif
 
-#ifdef PSP_ENABLE_PYTHON
-namespace py = boost::python;
-namespace np = boost::python::numpy;
-#include <perspective/numpy.h>
-#endif
-
 namespace perspective {
 
 t_lstore_recipe::t_lstore_recipe()
@@ -594,18 +588,14 @@ t_lstore::clone() const {
 }
 
 #ifdef PSP_ENABLE_PYTHON
-np::ndarray
+py::array
 t_lstore::_as_numpy(t_dtype dtype) {
     PSP_TRACE_SENTINEL();
     PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
     PSP_VERBOSE_ASSERT(dtype != DTYPE_STR, "as_numpy not implemented for string columns yet");
 
-    np::dtype npdtype = get_numpy_typenum_from_dtype(dtype);
-    py::tuple shape = py::make_tuple(m_size / get_dtype_size(dtype));
-    py::tuple stride = py::make_tuple(get_dtype_size(dtype));
-
-    np::ndarray result = np::from_data(m_base, npdtype, shape, stride, py::object());
-
+    // TODO
+    py::array result;
     return result;
     // PSP_VERBOSE_ASSERT(rval, "Null array found!");
 }

cpp/perspective/src/cpp/time.cpp

@@ -74,6 +74,14 @@ t_time::as_tm(struct tm& out) const {
     return gmtime(out, as_seconds(), 0) == 1;
 }
 
+std::tm
+t_time::get_tm() const {
+    std::tm rval;
+    gmtime(rval, microseconds(), 0);
+    std::cout << str(rval);
+    return rval;
+}
+
 std::int32_t
 isleap(long int year) {
     return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) ? 1 : 0;

cpp/perspective/src/include/perspective/binding.h

@@ -261,7 +261,7 @@ namespace binding {
     */
     template <typename T>
     std::tuple<std::string, std::string, std::vector<t_tscalar>> make_filter_term(
-        t_dtype column_type, T date_parser, const std::string column_name, const std::string& filter_op_str, T filter_term);
+        t_dtype column_type, T date_parser, const std::string& column_name, const std::string& filter_op_str, T filter_term);
     /**
      * @brief Create a `t_view_config` object from the binding language's `view_config` object.
      *

cpp/perspective/src/include/perspective/column.h

@@ -22,11 +22,6 @@
 #include <cmath>
 #include <tsl/hopscotch_map.h>
 
-#ifdef PSP_ENABLE_PYTHON
-namespace py = boost::python;
-namespace np = boost::python::numpy;
-#endif
-
 /*
 TODO -
 1. Implement implicit typepunning based on cardinality.
@@ -200,7 +195,7 @@ class PERSPECTIVE_EXPORT t_column {
     void borrow_vocabulary(const t_column& o);
 
 #ifdef PSP_ENABLE_PYTHON
-    np::ndarray _as_numpy();
+    py::array _as_numpy();
 #endif
 
 private:

cpp/perspective/src/include/perspective/date.h

@@ -14,6 +14,7 @@
 #include <perspective/utils.h>
 #include <perspective/exports.h>
 #include <boost/functional/hash.hpp>
+#include <ctime>
 #include <sstream>
 #include <string>
 #include <algorithm>
@@ -65,6 +66,13 @@ class PERSPECTIVE_EXPORT t_date {
     // only works for dates after 1900AD.)
     std::int32_t consecutive_day_idx() const;
 
+    /**
+     * @brief Return an `std::tm` object. `tm_hour`, `tm_min`, and `tm_sec` are set to 0.
+     * 
+     * @return std::tm 
+     */
+    std::tm get_tm() const;
+
     friend bool operator<(const t_date& a, const t_date& b);
     friend bool operator<=(const t_date& a, const t_date& b);
     friend bool operator>(const t_date& a, const t_date& b);

cpp/perspective/src/include/perspective/date_parser.h

@@ -9,8 +9,8 @@
 
 #pragma once
 #include <memory>
-#include <vector>
 #include <locale>
+#include <perspective/base.h>
 #include <perspective/first.h>
 #include <perspective/exports.h>
 

cpp/perspective/src/include/perspective/first.h

@@ -59,11 +59,8 @@
 #endif
 
 #ifdef PSP_ENABLE_PYTHON
-#include <boost/python.hpp>
-#include <boost/python/def.hpp>
-#include <boost/python/numpy.hpp>
-#include <numpy/arrayobject.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/numpy.h>
+namespace py = pybind11;
+#endif
 
-namespace py = boost::python;
-namespace np = boost::python::numpy;
-#endif

cpp/perspective/src/include/perspective/storage.h

@@ -17,10 +17,6 @@
 #include <perspective/debug_helpers.h>
 #include <cmath>
 
-#ifdef PSP_ENABLE_PYTHON
-namespace py = boost::python;
-namespace np = boost::python::numpy;
-#endif
 
 /*
 TODO.
@@ -210,7 +206,7 @@ class PERSPECTIVE_MPROTECT_EXPORT t_lstore : public t_debug_helper
 
 #ifdef PSP_ENABLE_PYTHON
     /* Python bits */
-    np::ndarray _as_numpy(t_dtype dtype);
+    py::array _as_numpy(t_dtype dtype);
 #endif
 
 protected:

cpp/perspective/src/include/perspective/time.h

@@ -13,6 +13,7 @@
 #include <perspective/raw_types.h>
 #include <perspective/exports.h>
 #include <boost/functional/hash.hpp>
+#include <chrono>
 #include <iostream>
 #include <sstream>
 #include <iomanip>
@@ -87,6 +88,13 @@ class PERSPECTIVE_EXPORT t_time {
 
     bool as_tm(struct tm& out) const;
 
+    /**
+     * @brief Return the instance as an std::tm object.
+     * 
+     * @return std::tm 
+     */
+    std::tm get_tm() const;
+
     std::int32_t gmtime(struct tm& out, std::int64_t secs, std::int32_t offset) const;
 
     std::int32_t year(const struct tm& t) const;