Skip to content

Commit

Permalink
1. Add imperative gperf profiler
Browse files Browse the repository at this point in the history
2. Add binutils 2.27 in manylinux support

test=develop
  • Loading branch information
velconia committed Mar 14, 2019
1 parent 98dfb49 commit 7355d41
Show file tree
Hide file tree
Showing 8 changed files with 156 additions and 40 deletions.
12 changes: 2 additions & 10 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ message(STATUS "CXX compiler: ${CMAKE_CXX_COMPILER}, version: "
"${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
message(STATUS "C compiler: ${CMAKE_C_COMPILER}, version: "
"${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
message(STATUS "AR tools: ${CMAKE_AR}")

if(WIN32)
set(CMAKE_SUPPRESS_REGENERATION ON)
set(CMAKE_STATIC_LIBRARY_PREFIX lib)
Expand All @@ -37,16 +39,6 @@ if(WIN32)
set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} ${PADDLE_LINK_FLAGS}")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${PADDLE_LINK_FLAGS}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${PADDLE_LINK_FLAGS}")
# else()
# set(CMAKE_C_ARCHIVE_CREATE "<CMAKE_AR> --target elf64-x86-64 cr <TARGET> <LINK_FLAGS> <OBJECTS>")
# set(CMAKE_C_ARCHIVE_APPEND "<CMAKE_AR> --target elf64-x86-64 r <TARGET> <LINK_FLAGS> <OBJECTS>")
# # set(CMAKE_C_ARCHIVE_FINISH "<CMAKE_RANLIB> --enable-64-bit-archive <TARGET>")
# set(CMAKE_CXX_ARCHIVE_CREATE ${CMAKE_C_ARCHIVE_CREATE})
# set(CMAKE_CXX_ARCHIVE_APPEND ${CMAKE_C_ARCHIVE_APPEND})
# # set(CMAKE_CXX_ARCHIVE_FINISH ${CMAKE_C_ARCHIVE_FINISH})
# set(CMAKE_Fortran_ARCHIVE_CREATE ${CMAKE_C_ARCHIVE_CREATE})
# set(CMAKE_Fortran_ARCHIVE_APPEND ${CMAKE_C_ARCHIVE_APPEND})
# # set(CMAKE_Fortran_ARCHIVE_FINISH ${CMAKE_C_ARCHIVE_FINISH})
endif(WIN32)

find_package(CUDA QUIET)
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ add_subdirectory(pybind)

# NOTE: please add subdirectory inference at last.
add_subdirectory(inference)
#add_subdirectory(train)
add_subdirectory(train)
65 changes: 65 additions & 0 deletions paddle/fluid/imperative/profiler.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/fluid/imperative/profiler.h"

#ifdef WITH_GPERFTOOLS
#include "gperftools/profiler.h"
#endif
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <mutex> // NOLINT
#include <thread> // NOLINT

DEFINE_string(
tracer_profile_fname, "xxgperf",
"Profiler filename for imperative tracer, which generated by gperftools."
"Only valid when compiled `WITH_PROFILER=ON`. Empty if disable.");

namespace paddle {
namespace imperative {

static std::once_flag gTracerProfileOnce;
#ifdef WITH_GPERFTOOLS
static bool gTracerProfilerStarted = false;
#endif

void StartProfile() {
LOG(ERROR) << "XX " << FLAGS_tracer_profile_fname;
if (!FLAGS_tracer_profile_fname.empty()) {
std::call_once(gTracerProfileOnce, [] {
#ifdef WITH_GPERFTOOLS
ProfilerStart(FLAGS_tracer_profile_fname.c_str());
gTracerProfilerStarted = true;
LOG(ERROR) << "YY";
#else
LOG(WARNING) << "Paddle is not compiled with gperftools. "
"FLAGS_tracer_profile_fname will be ignored";
#endif
});
}
}

void StopProfile() {
LOG(ERROR) << "ZZ " << FLAGS_tracer_profile_fname;
#ifdef WITH_GPERFTOOLS
ProfilerFlush();
#else
LOG(WARNING) << "Paddle is not compiled with gperftools. "
"FLAGS_tracer_profile_fname will be ignored";
#endif
}

} // namespace imperative
} // namespace paddle
25 changes: 25 additions & 0 deletions paddle/fluid/imperative/profiler.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once

namespace paddle {
namespace imperative {

extern void StartProfile();

extern void StopProfile();

} // namespace imperative
} // namespace paddle
51 changes: 25 additions & 26 deletions paddle/fluid/inference/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,12 @@ if (WIN32)
list(APPEND fluid_third_partys gflags glog protobuf cblas)
endif(WIN32)

# # paddle_fluid_origin exclude inference api interface
# if(WIN32)
# sep_library(paddle_fluid_origin DEPS ${fluid_modules} paddle_fluid_api)
# else(WIN32)
# set(CMAKE_C_ARCHIVE_CREATE "<CMAKE_AR> --target elf64-x86-64 cr paddle_fluid_origin ${fluid_modules} paddle_fluid_api")
# cc_library(paddle_fluid_origin DEPS ${fluid_modules} paddle_fluid_api)
# endif(WIN32)
# paddle_fluid_origin exclude inference api interface
if(WIN32)
sep_library(paddle_fluid_origin DEPS ${fluid_modules} paddle_fluid_api)
else(WIN32)
cc_library(paddle_fluid_origin DEPS ${fluid_modules} paddle_fluid_api)
endif(WIN32)

add_subdirectory(api)

Expand All @@ -41,19 +40,19 @@ set(SHARED_INFERENCE_SRCS
${CMAKE_CURRENT_SOURCE_DIR}/api/analysis_predictor.cc
${CMAKE_CURRENT_SOURCE_DIR}/api/details/zero_copy_tensor.cc)

# if(WIN32)
# sep_library(paddle_fluid DEPS ${fluid_modules} ${STATIC_INFERENCE_APIS} zero_copy_tensor reset_tensor_array
# analysis_config paddle_pass_builder)
# else(WIN32)
# cc_library(paddle_fluid DEPS ${fluid_modules} ${STATIC_INFERENCE_APIS}
# zero_copy_tensor reset_tensor_array analysis_config paddle_pass_builder)
# endif(WIN32)
if(WIN32)
sep_library(paddle_fluid DEPS ${fluid_modules} ${STATIC_INFERENCE_APIS} zero_copy_tensor reset_tensor_array
analysis_config paddle_pass_builder)
else(WIN32)
cc_library(paddle_fluid DEPS ${fluid_modules} ${STATIC_INFERENCE_APIS}
zero_copy_tensor reset_tensor_array analysis_config paddle_pass_builder)
endif(WIN32)

# if(NOT APPLE)
# # TODO(liuyiqu: Temporarily disable the link flag because it is not support on Mac.
# set(LINK_FLAGS "-Wl,--retain-symbols-file ${CMAKE_CURRENT_SOURCE_DIR}/paddle_fluid.sym")
# set_target_properties(paddle_fluid PROPERTIES LINK_FLAGS "${LINK_FLAGS}")
# endif()
if(NOT APPLE)
# TODO(liuyiqu: Temporarily disable the link flag because it is not support on Mac.
set(LINK_FLAGS "-Wl,--retain-symbols-file ${CMAKE_CURRENT_SOURCE_DIR}/paddle_fluid.sym")
set_target_properties(paddle_fluid PROPERTIES LINK_FLAGS "${LINK_FLAGS}")
endif()

# Create shared library
if(WIN32)
Expand Down Expand Up @@ -87,10 +86,10 @@ if(NOT APPLE AND NOT WIN32)
add_custom_target(check_symbol ALL DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/.check_symbol")
endif()

# if(WITH_TESTING)
# # tests/book depends the models that generated by python/paddle/fluid/tests/book
# add_subdirectory(tests/book)
# if(WITH_INFERENCE_API_TEST)
# add_subdirectory(tests/api)
# endif()
# endif()
if(WITH_TESTING)
# tests/book depends the models that generated by python/paddle/fluid/tests/book
add_subdirectory(tests/book)
if(WITH_INFERENCE_API_TEST)
add_subdirectory(tests/api)
endif()
endif()
5 changes: 2 additions & 3 deletions paddle/fluid/pybind/pybind.cc
Original file line number Diff line number Diff line change
Expand Up @@ -149,11 +149,10 @@ PYBIND11_MODULE(core, m) {
m.def("print_mem_usage",
[]() { return memory::allocation::GPUMemMonitor.PrintMemUsage(); });

m.def("start_imperative_profiler",
m.def("start_imperative_gperf_profiler",
[]() { imperative::StartProfile(); });

m.def("stop_imperative_profiler",
[]() { imperative::StopProfile(); });
m.def("stop_imperative_gperf_profiler", []() { imperative::StopProfile(); });

py::class_<imperative::VarBase>(m, "VarBase", R"DOC()DOC")
.def(
Expand Down
30 changes: 30 additions & 0 deletions python/paddle/fluid/imperative/profiler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

from .. import core

__all__ = [
'start_gperf_profiler',
'stop_gperf_profiler',
]


def start_gperf_profiler():
core.start_imperative_gperf_profiler()


def stop_gperf_profiler():
core.stop_imperative_gperf_profiler()
6 changes: 6 additions & 0 deletions tools/manylinux1/build_scripts/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -153,3 +153,9 @@ done

# Restore LD_LIBRARY_PATH
LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}"

# According to ar issues: https://lists.gnu.org/archive/html/bug-binutils/2016-05/msg00211.html
# we should install new version ar with 64-bit supported here
wget https://ftp.gnu.org/gnu/binutils/binutils-2.27.tar.gz
tar xzf binutils-2.27.tar.gz && cd binutils-2.27
./configure --prefix=/opt/rh/devtoolset-2/root/usr/ --enable-64-bit-archive && make -j `nproc` && make install

0 comments on commit 7355d41

Please sign in to comment.