diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
index 6e8cda25203..0886a8b0a52 100644
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(cuda_rtc)
 add_subdirectory(db)
 add_subdirectory(distributed)
 # add_subdirectory(experiments) # note, we may remove this folder at some point
+add_subdirectory(ideep)
 add_subdirectory(image)
 add_subdirectory(video)
 add_subdirectory(mkl)
diff --git a/caffe2/contrib/CMakeLists.txt b/caffe2/contrib/CMakeLists.txt
index 5351bbabf74..7becad2b0af 100644
--- a/caffe2/contrib/CMakeLists.txt
+++ b/caffe2/contrib/CMakeLists.txt
@@ -1,6 +1,5 @@
 add_subdirectory(aten)
 add_subdirectory(gloo)
-add_subdirectory(ideep)
 add_subdirectory(nccl)
 add_subdirectory(opencl)
 add_subdirectory(prof)
diff --git a/caffe2/contrib/ideep/CMakeLists.txt b/caffe2/ideep/CMakeLists.txt
similarity index 100%
rename from caffe2/contrib/ideep/CMakeLists.txt
rename to caffe2/ideep/CMakeLists.txt
diff --git a/caffe2/contrib/ideep/ideep_utils.h b/caffe2/ideep/ideep_utils.h
similarity index 87%
rename from caffe2/contrib/ideep/ideep_utils.h
rename to caffe2/ideep/ideep_utils.h
index 82e5aab4961..aba76a60aa6 100644
--- a/caffe2/contrib/ideep/ideep_utils.h
+++ b/caffe2/ideep/ideep_utils.h
@@ -4,8 +4,8 @@
 
 // All caffe2 ideep related headers
 #include <ideep.hpp>
-#include <caffe2/contrib/ideep/utils/ideep_context.h>
-#include <caffe2/contrib/ideep/utils/ideep_operator.h>
+#include <caffe2/ideep/utils/ideep_context.h>
+#include <caffe2/ideep/utils/ideep_operator.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/concat_split_op.cc b/caffe2/ideep/operators/concat_split_op.cc
similarity index 99%
rename from caffe2/contrib/ideep/operators/concat_split_op.cc
rename to caffe2/ideep/operators/concat_split_op.cc
index fe4f06c31ce..f589185caa0 100644
--- a/caffe2/contrib/ideep/operators/concat_split_op.cc
+++ b/caffe2/ideep/operators/concat_split_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/conv_fusion_op.cc b/caffe2/ideep/operators/conv_fusion_op.cc
similarity index 98%
rename from caffe2/contrib/ideep/operators/conv_fusion_op.cc
rename to caffe2/ideep/operators/conv_fusion_op.cc
index ea820fff9b4..e3d0b6c8d93 100644
--- a/caffe2/contrib/ideep/operators/conv_fusion_op.cc
+++ b/caffe2/ideep/operators/conv_fusion_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/operators/conv_pool_base_op.h>
+#include <caffe2/ideep/operators/conv_pool_base_op.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/conv_op.cc b/caffe2/ideep/operators/conv_op.cc
similarity index 96%
rename from caffe2/contrib/ideep/operators/conv_op.cc
rename to caffe2/ideep/operators/conv_op.cc
index 554818b8e88..ae11352ef24 100644
--- a/caffe2/contrib/ideep/operators/conv_op.cc
+++ b/caffe2/ideep/operators/conv_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/operators/conv_pool_base_op.h>
+#include <caffe2/ideep/operators/conv_pool_base_op.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/conv_pool_base_op.h b/caffe2/ideep/operators/conv_pool_base_op.h
similarity index 98%
rename from caffe2/contrib/ideep/operators/conv_pool_base_op.h
rename to caffe2/ideep/operators/conv_pool_base_op.h
index e5a2d7720bc..824ac36bd89 100644
--- a/caffe2/contrib/ideep/operators/conv_pool_base_op.h
+++ b/caffe2/ideep/operators/conv_pool_base_op.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 #include <caffe2/operators/conv_pool_op_base.h>
 
 namespace caffe2 {
diff --git a/caffe2/contrib/ideep/operators/dropout_op.cc b/caffe2/ideep/operators/dropout_op.cc
similarity index 95%
rename from caffe2/contrib/ideep/operators/dropout_op.cc
rename to caffe2/ideep/operators/dropout_op.cc
index d0b621e2d43..baeef0a590f 100644
--- a/caffe2/contrib/ideep/operators/dropout_op.cc
+++ b/caffe2/ideep/operators/dropout_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/elementwise_sum_op.cc b/caffe2/ideep/operators/elementwise_sum_op.cc
similarity index 95%
rename from caffe2/contrib/ideep/operators/elementwise_sum_op.cc
rename to caffe2/ideep/operators/elementwise_sum_op.cc
index ae991154da4..2f0bbf3bebd 100644
--- a/caffe2/contrib/ideep/operators/elementwise_sum_op.cc
+++ b/caffe2/ideep/operators/elementwise_sum_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/fully_connected_op.cc b/caffe2/ideep/operators/fully_connected_op.cc
similarity index 95%
rename from caffe2/contrib/ideep/operators/fully_connected_op.cc
rename to caffe2/ideep/operators/fully_connected_op.cc
index 15ba82d5852..5f2b770d0ed 100644
--- a/caffe2/contrib/ideep/operators/fully_connected_op.cc
+++ b/caffe2/ideep/operators/fully_connected_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/local_response_normalization_op.cc b/caffe2/ideep/operators/local_response_normalization_op.cc
similarity index 95%
rename from caffe2/contrib/ideep/operators/local_response_normalization_op.cc
rename to caffe2/ideep/operators/local_response_normalization_op.cc
index 60b44ef1969..8a36f0294b3 100644
--- a/caffe2/contrib/ideep/operators/local_response_normalization_op.cc
+++ b/caffe2/ideep/operators/local_response_normalization_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/operator_fallback_ideep.cc b/caffe2/ideep/operators/operator_fallback_ideep.cc
similarity index 90%
rename from caffe2/contrib/ideep/operators/operator_fallback_ideep.cc
rename to caffe2/ideep/operators/operator_fallback_ideep.cc
index 60ffdbee0d3..4a3be8d483c 100644
--- a/caffe2/contrib/ideep/operators/operator_fallback_ideep.cc
+++ b/caffe2/ideep/operators/operator_fallback_ideep.cc
@@ -1,5 +1,5 @@
-#include <caffe2/contrib/ideep/operators/operator_fallback_ideep.h>
-#include <caffe2/contrib/ideep/utils/ideep_operator.h>
+#include <caffe2/ideep/operators/operator_fallback_ideep.h>
+#include <caffe2/ideep/utils/ideep_operator.h>
 
 #include <caffe2/operators/cross_entropy_op.h>
 #include <caffe2/operators/filler_op.h>
diff --git a/caffe2/contrib/ideep/operators/operator_fallback_ideep.h b/caffe2/ideep/operators/operator_fallback_ideep.h
similarity index 99%
rename from caffe2/contrib/ideep/operators/operator_fallback_ideep.h
rename to caffe2/ideep/operators/operator_fallback_ideep.h
index 1687c9be67a..a52b002b2a9 100644
--- a/caffe2/contrib/ideep/operators/operator_fallback_ideep.h
+++ b/caffe2/ideep/operators/operator_fallback_ideep.h
@@ -3,7 +3,7 @@
 #include <caffe2/core/common.h>
 #include <caffe2/core/context.h>
 #include <caffe2/core/operator.h>
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 #include <caffe2/proto/caffe2.pb.h>
 
 namespace caffe2 {
diff --git a/caffe2/contrib/ideep/operators/pool_op.cc b/caffe2/ideep/operators/pool_op.cc
similarity index 95%
rename from caffe2/contrib/ideep/operators/pool_op.cc
rename to caffe2/ideep/operators/pool_op.cc
index 3ddc6de4873..abdfb3f86c7 100644
--- a/caffe2/contrib/ideep/operators/pool_op.cc
+++ b/caffe2/ideep/operators/pool_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/operators/conv_pool_base_op.h>
+#include <caffe2/ideep/operators/conv_pool_base_op.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/relu_op.cc b/caffe2/ideep/operators/relu_op.cc
similarity index 92%
rename from caffe2/contrib/ideep/operators/relu_op.cc
rename to caffe2/ideep/operators/relu_op.cc
index d7499131849..fc267d06c94 100644
--- a/caffe2/contrib/ideep/operators/relu_op.cc
+++ b/caffe2/ideep/operators/relu_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/operators/spatial_batch_norm_op.cc b/caffe2/ideep/operators/spatial_batch_norm_op.cc
similarity index 97%
rename from caffe2/contrib/ideep/operators/spatial_batch_norm_op.cc
rename to caffe2/ideep/operators/spatial_batch_norm_op.cc
index 7bf4df12d9b..0d3a1b0401c 100644
--- a/caffe2/contrib/ideep/operators/spatial_batch_norm_op.cc
+++ b/caffe2/ideep/operators/spatial_batch_norm_op.cc
@@ -1,4 +1,4 @@
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 
 namespace caffe2 {
 
diff --git a/caffe2/contrib/ideep/utils/ideep_context.h b/caffe2/ideep/utils/ideep_context.h
similarity index 100%
rename from caffe2/contrib/ideep/utils/ideep_context.h
rename to caffe2/ideep/utils/ideep_context.h
diff --git a/caffe2/contrib/ideep/utils/ideep_operator.h b/caffe2/ideep/utils/ideep_operator.h
similarity index 100%
rename from caffe2/contrib/ideep/utils/ideep_operator.h
rename to caffe2/ideep/utils/ideep_operator.h
diff --git a/caffe2/contrib/ideep/utils/ideep_register.cc b/caffe2/ideep/utils/ideep_register.cc
similarity index 100%
rename from caffe2/contrib/ideep/utils/ideep_register.cc
rename to caffe2/ideep/utils/ideep_register.cc
diff --git a/caffe2/python/pybind_state_ideep.cc b/caffe2/python/pybind_state_ideep.cc
index 43f00cded29..952327f13c1 100644
--- a/caffe2/python/pybind_state_ideep.cc
+++ b/caffe2/python/pybind_state_ideep.cc
@@ -9,7 +9,7 @@
 #include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 
-#include <caffe2/contrib/ideep/ideep_utils.h>
+#include <caffe2/ideep/ideep_utils.h>
 
 namespace caffe2 {
 namespace python {