1

I'm writing an inference code to load a converted pytorch model (a tagging model from imagenet) in C++. I used c++ pytorch frontend API. My code works correctly on CPU but does not work on GPU. The problem is that when I want to print the final results, I get the Segmentation fault (core dumped) error. I have to transfer the "top_scores_a" and "top_idx_a" variables to the CPU but I don't know how to do so.

I load the model and the input image on GPU. The error is occurred in the following part:

for (int i = 0; i < 2; ++i)
    {
        // int idx = top_idxs_a[i];
        std::cout << "top-" << i+1 << " label: ";
        // std::cout << labels[idx] << ", score: " << top_scores_a[i] << std::endl;
    }

The complete code is available here:

#include "torch/script.h"
#include <torch/script.h>
#include <torch/torch.h>
#include <ATen/Tensor.h>
#include <opencv2/opencv.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <time.h> 

#include <iostream>
#include <memory>
#include <cuda.h>
#include <cuda_runtime_api.h>

using namespace std;



// __global__
int main(int argc, const char* argv[]) {

    //// asign gpu
    torch::Device device(torch::kCPU);
    clock_t tStart = clock();

    //// check cuda visibility
    if (torch::cuda::is_available()) 
    {
        std::cout << "CUDA is available! Run on GPU." << std::endl;
        device = torch::kCUDA;

    }

    if (argc != 4) {
        cout << "ptcpp path/to/scripts/model.pt path/to/image.jpg path/to/label.txt\n";
        return -1;
    }

    cout << "Will load from " << argv[1] << endl;
    shared_ptr<torch::jit::script::Module> module = torch::jit::load(argv[1]);
    module->to(device); // on gpu

    if (module == nullptr) {
        cerr << "model load error from " << argv[1] << endl;
    }
    cout << "Model load ok.\n";

    // load image and transform
    cv::Mat image;
    image = cv::imread(argv[2], 1);

    cv::Mat image_rgb;
    cv::cvtColor(image, image_rgb, CV_BGR2RGB);  

    cv::Mat image_resized;
    cv::resize(image_rgb, image_resized, cv::Size(224, 224));

    cv::Mat image_resized_float;
    image_resized.convertTo(image_resized_float, CV_32F, 1.0/255);

    auto img_tensor = torch::CPU(torch::kFloat32).tensorFromBlob(image_resized_float.data, {1, 224, 224, 3}).to(device); // work correctly

    cout << "img tensor loaded..\n";
    img_tensor = img_tensor.permute({0, 3, 1, 2});
    img_tensor[0][0] = img_tensor[0][0].sub(0.485).div(0.229);
    img_tensor[0][1] = img_tensor[0][1].sub(0.456).div(0.224);
    img_tensor[0][2] = img_tensor[0][2].sub(0.406).div(0.225);

    auto img_var = torch::autograd::make_variable(img_tensor, false);

    vector<torch::jit::IValue> inputs;
    inputs.push_back(img_var);
    torch::Tensor out_tensor = module->forward(inputs).toTensor();


    // load labels
    vector<string> labels;
    ifstream ins;
    ins.open(argv[3]);
    string line;
    while (getline(ins, line)) 
    {
        labels.push_back(line);
    }


    std::tuple<torch::Tensor,torch::Tensor> result = out_tensor.sort(-1, true); //-1
    torch::Tensor top_scores = std::get<0>(result)[0];
    torch::Tensor top_idxs = std::get<1>(result)[0].toType(torch::kInt32);

    auto top_scores_a = top_scores.accessor<float,1>();
    auto top_idxs_a = top_idxs.accessor<int,1>();


    for (int i = 0; i < 2; ++i)
    {
        int idx = top_idxs_a[i];
        std::cout << "top-" << i+1 << " label: ";
        std::cout << labels[idx] << ", score: " << top_scores_a[i] << std::endl;
    }


    float tend = clock();
    printf("Time taken: %.2fs\n", (double)(tend - tStart)/CLOCKS_PER_SEC);

    return 0;
}

1 Answer 1

-1

To move data from CPU to GPU and vice versa you must allocate so called managed memory. Have a look here for some example code https://devblogs.nvidia.com/even-easier-introduction-cuda

If your cuda version does not support cudaMallocManaged, then you have to use cudaMalloc + cudaMemcpy sequence.

4
  • Thanks. but this document is about running codes on GPUs. My code is already on GPU. I want to move "top_scores_a" on CPU to use it.
    – Mahsa
    Jan 25, 2019 at 15:36
  • 2
    your variable "top_scores_a" must be allocated in the way both CPU and GPU could access it. If you have a managed pointer then this pointer could be accesses from GPU code and CPU code smoothly. For you it will be a simple pointer.
    – Dmitry
    Jan 25, 2019 at 15:40
  • As I want to allocate memory, I need to know the type of "top_scores_a" which is "auto". When I check its type, I get this : "N2at14TensorAccessorIfLm1ENS_16DefaultPtrTraitsElEE". So, how should I declare the type of memory in the allocation step?
    – Mahsa
    Jan 25, 2019 at 18:39
  • Your type is exactly the type top_scores.accessor() returns.have a look at the declaration of thus function
    – Dmitry
    Jan 25, 2019 at 21:14

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.