Multiple inputs for plugin

AI & Data Science Deep Learning (Training & Inference) TensorRT
1

For multiple inputs in plugin, what should be the source for

void copyDeviceInputToFP32(const void* src, void*& dst){}?

Sample has only one source as follows. If I have two sources, how to implement src and dst?

    void copyDeviceInputToFP32(const void* src, void*& dst)
    {
        assert(mDataType == DataType::kINT8);
        size_t inCount = getC(mInputDims) * getH(mInputDims) * getW(mInputDims);
        std::unique_ptr<char> inputTmp{new char[inCount * elementSize(mDataType)]};
        CHECK(cudaMemcpy(inputTmp.get(), src, inCount * elementSize(mDataType), cudaMemcpyDeviceToHost));
        std::unique_ptr<float> inputFP32{new float[inCount]};
        transform<DataType::kINT8, DataType::kFLOAT>(inputTmp.get(), inputFP32.get(), inCount);
        // int8 scale
        int hw = mInputDims.d[1] * mInputDims.d[2];
        for (int j = 0; j < mInputDims.d[0]; ++j)
        {
            std::transform(inputFP32.get() + hw * j, inputFP32.get() + hw * (j + 1), inputFP32.get() + hw * j,
                [&](float in) -> float { return in * mInHostScale; });
        }
        CHECK(cudaMalloc(&dst, inCount * elementSize(DataType::kFLOAT)));
        CHECK(cudaMemcpy(dst, inputFP32.get(), inCount * elementSize(DataType::kFLOAT), cudaMemcpyHostToDevice));
    }
2

Hi @edit_or
I am not sure which sample you are referring to?
Please take reference from the sample plugin link
https://github.com/NVIDIA/TensorRT/tree/master/samples/opensource/samplePlugin

Thanks!

3

They are from IPluginV2IOExt sample.

There are two methods for Int8 conversions.

    void copyDeviceInputToFP32(const void* src, void*& dst)
    {
        assert(mDataType == DataType::kINT8);
        size_t inCount = getC(mInputDims) * getH(mInputDims) * getW(mInputDims);
        std::unique_ptr<char> inputTmp{new char[inCount * elementSize(mDataType)]};
        CHECK(cudaMemcpy(inputTmp.get(), src, inCount * elementSize(mDataType), cudaMemcpyDeviceToHost));
        std::unique_ptr<float> inputFP32{new float[inCount]};
        transform<DataType::kINT8, DataType::kFLOAT>(inputTmp.get(), inputFP32.get(), inCount);
        // int8 scale
        int hw = mInputDims.d[1] * mInputDims.d[2];
        for (int j = 0; j < mInputDims.d[0]; ++j)
        {
            std::transform(inputFP32.get() + hw * j, inputFP32.get() + hw * (j + 1), inputFP32.get() + hw * j,
                [&](float in) -> float { return in * mInHostScale; });
        }
        CHECK(cudaMalloc(&dst, inCount * elementSize(DataType::kFLOAT)));
        CHECK(cudaMemcpy(dst, inputFP32.get(), inCount * elementSize(DataType::kFLOAT), cudaMemcpyHostToDevice));
    }

    void copyDeviceToInt8Output(const void* src, void* dst)
    {
        size_t outCount = getC(mOutputDims) * getH(mOutputDims) * getW(mOutputDims);
        std::unique_ptr<float> outTmp{new float[outCount]};
        CHECK(cudaMemcpy(outTmp.get(), src, outCount * elementSize(DataType::kFLOAT), cudaMemcpyDeviceToHost));
        std::unique_ptr<char> outInt8{new char[outCount * elementSize(DataType::kINT8)]};
        // int8 + scale
        int hw = mOutputDims.d[1] * mOutputDims.d[2];
        for (int j = 0; j < mInputDims.d[0]; ++j)
        {
            std::transform(outTmp.get() + hw * j, outTmp.get() + hw * (j + 1), outTmp.get() + hw * j,
                [&](float in) -> float { return in / mOutHostScale; });
        }
        transform<DataType::kFLOAT, DataType::kINT8>(outTmp.get(), outInt8.get(), outCount);
        CHECK(cudaMemcpy(dst, outInt8.get(), outCount, cudaMemcpyHostToDevice));
    }

How can I manage src and dst in the conversions if I have two sources?

4

May I have reply for this?

5

Hi @edit_or,
Are you still facing the issue?