vectorization_logic.cpp 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. // This file is part of Eigen, a lightweight C++ template library
  2. // for linear algebra.
  3. //
  4. // Copyright (C) 2015 Gael Guennebaud <gael.guennebaud@inria.fr>
  5. //
  6. // This Source Code Form is subject to the terms of the Mozilla
  7. // Public License v. 2.0. If a copy of the MPL was not distributed
  8. // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. #ifdef EIGEN_TEST_PART_1
  10. #define EIGEN_UNALIGNED_VECTORIZE 1
  11. #endif
  12. #ifdef EIGEN_TEST_PART_2
  13. #define EIGEN_UNALIGNED_VECTORIZE 0
  14. #endif
  15. #ifdef EIGEN_DEFAULT_TO_ROW_MAJOR
  16. #undef EIGEN_DEFAULT_TO_ROW_MAJOR
  17. #endif
  18. #define EIGEN_DEBUG_ASSIGN
  19. #include "main.h"
  20. #include <typeinfo>
  21. // Disable "ignoring attributes on template argument"
  22. // for packet_traits<Packet*>
  23. // => The only workaround would be to wrap _m128 and the likes
  24. // within wrappers.
  25. #if EIGEN_GNUC_AT_LEAST(6,0)
  26. #pragma GCC diagnostic ignored "-Wignored-attributes"
  27. #endif
  28. using internal::demangle_flags;
  29. using internal::demangle_traversal;
  30. using internal::demangle_unrolling;
  31. template<typename Dst, typename Src>
  32. bool test_assign(const Dst&, const Src&, int traversal, int unrolling)
  33. {
  34. EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src);
  35. typedef internal::copy_using_evaluator_traits<internal::evaluator<Dst>,internal::evaluator<Src>, internal::assign_op<typename Dst::Scalar,typename Src::Scalar> > traits;
  36. bool res = traits::Traversal==traversal;
  37. if(unrolling==InnerUnrolling+CompleteUnrolling)
  38. res = res && (int(traits::Unrolling)==InnerUnrolling || int(traits::Unrolling)==CompleteUnrolling);
  39. else
  40. res = res && int(traits::Unrolling)==unrolling;
  41. if(!res)
  42. {
  43. std::cerr << "Src: " << demangle_flags(Src::Flags) << std::endl;
  44. std::cerr << " " << demangle_flags(internal::evaluator<Src>::Flags) << std::endl;
  45. std::cerr << "Dst: " << demangle_flags(Dst::Flags) << std::endl;
  46. std::cerr << " " << demangle_flags(internal::evaluator<Dst>::Flags) << std::endl;
  47. traits::debug();
  48. std::cerr << " Expected Traversal == " << demangle_traversal(traversal)
  49. << " got " << demangle_traversal(traits::Traversal) << "\n";
  50. std::cerr << " Expected Unrolling == " << demangle_unrolling(unrolling)
  51. << " got " << demangle_unrolling(traits::Unrolling) << "\n";
  52. }
  53. return res;
  54. }
  55. template<typename Dst, typename Src>
  56. bool test_assign(int traversal, int unrolling)
  57. {
  58. EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src);
  59. typedef internal::copy_using_evaluator_traits<internal::evaluator<Dst>,internal::evaluator<Src>, internal::assign_op<typename Dst::Scalar,typename Src::Scalar> > traits;
  60. bool res = traits::Traversal==traversal && traits::Unrolling==unrolling;
  61. if(!res)
  62. {
  63. std::cerr << "Src: " << demangle_flags(Src::Flags) << std::endl;
  64. std::cerr << " " << demangle_flags(internal::evaluator<Src>::Flags) << std::endl;
  65. std::cerr << "Dst: " << demangle_flags(Dst::Flags) << std::endl;
  66. std::cerr << " " << demangle_flags(internal::evaluator<Dst>::Flags) << std::endl;
  67. traits::debug();
  68. std::cerr << " Expected Traversal == " << demangle_traversal(traversal)
  69. << " got " << demangle_traversal(traits::Traversal) << "\n";
  70. std::cerr << " Expected Unrolling == " << demangle_unrolling(unrolling)
  71. << " got " << demangle_unrolling(traits::Unrolling) << "\n";
  72. }
  73. return res;
  74. }
  75. template<typename Xpr>
  76. bool test_redux(const Xpr&, int traversal, int unrolling)
  77. {
  78. typedef typename Xpr::Scalar Scalar;
  79. typedef internal::redux_traits<internal::scalar_sum_op<Scalar,Scalar>,internal::redux_evaluator<Xpr> > traits;
  80. bool res = traits::Traversal==traversal && traits::Unrolling==unrolling;
  81. if(!res)
  82. {
  83. std::cerr << demangle_flags(Xpr::Flags) << std::endl;
  84. std::cerr << demangle_flags(internal::evaluator<Xpr>::Flags) << std::endl;
  85. traits::debug();
  86. std::cerr << " Expected Traversal == " << demangle_traversal(traversal)
  87. << " got " << demangle_traversal(traits::Traversal) << "\n";
  88. std::cerr << " Expected Unrolling == " << demangle_unrolling(unrolling)
  89. << " got " << demangle_unrolling(traits::Unrolling) << "\n";
  90. }
  91. return res;
  92. }
  93. template<typename Scalar, bool Enable = internal::packet_traits<Scalar>::Vectorizable>
  94. struct vectorization_logic
  95. {
  96. typedef internal::packet_traits<Scalar> PacketTraits;
  97. typedef typename internal::packet_traits<Scalar>::type PacketType;
  98. typedef typename internal::unpacket_traits<PacketType>::half HalfPacketType;
  99. enum {
  100. PacketSize = internal::unpacket_traits<PacketType>::size,
  101. HalfPacketSize = internal::unpacket_traits<HalfPacketType>::size
  102. };
  103. static void run()
  104. {
  105. typedef Matrix<Scalar,PacketSize,1> Vector1;
  106. typedef Matrix<Scalar,Dynamic,1> VectorX;
  107. typedef Matrix<Scalar,Dynamic,Dynamic> MatrixXX;
  108. typedef Matrix<Scalar,PacketSize,PacketSize> Matrix11;
  109. typedef Matrix<Scalar,(Matrix11::Flags&RowMajorBit)?8:2*PacketSize,(Matrix11::Flags&RowMajorBit)?2*PacketSize:8> Matrix22;
  110. typedef Matrix<Scalar,(Matrix11::Flags&RowMajorBit)?16:4*PacketSize,(Matrix11::Flags&RowMajorBit)?4*PacketSize:16> Matrix44;
  111. typedef Matrix<Scalar,(Matrix11::Flags&RowMajorBit)?16:4*PacketSize,(Matrix11::Flags&RowMajorBit)?4*PacketSize:16,DontAlign|EIGEN_DEFAULT_MATRIX_STORAGE_ORDER_OPTION> Matrix44u;
  112. typedef Matrix<Scalar,4*PacketSize,4*PacketSize,ColMajor> Matrix44c;
  113. typedef Matrix<Scalar,4*PacketSize,4*PacketSize,RowMajor> Matrix44r;
  114. typedef Matrix<Scalar,
  115. (PacketSize==16 ? 8 : PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1),
  116. (PacketSize==16 ? 2 : PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1)
  117. > Matrix1;
  118. typedef Matrix<Scalar,
  119. (PacketSize==16 ? 8 : PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1),
  120. (PacketSize==16 ? 2 : PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1),
  121. DontAlign|((Matrix1::Flags&RowMajorBit)?RowMajor:ColMajor)> Matrix1u;
  122. // this type is made such that it can only be vectorized when viewed as a linear 1D vector
  123. typedef Matrix<Scalar,
  124. (PacketSize==16 ? 4 : PacketSize==8 ? 4 : PacketSize==4 ? 6 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?2:3) : /*PacketSize==1 ?*/ 1),
  125. (PacketSize==16 ? 12 : PacketSize==8 ? 6 : PacketSize==4 ? 2 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?3:2) : /*PacketSize==1 ?*/ 3)
  126. > Matrix3;
  127. #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT
  128. VERIFY(test_assign(Vector1(),Vector1(),
  129. InnerVectorizedTraversal,CompleteUnrolling));
  130. VERIFY(test_assign(Vector1(),Vector1()+Vector1(),
  131. InnerVectorizedTraversal,CompleteUnrolling));
  132. VERIFY(test_assign(Vector1(),Vector1().cwiseProduct(Vector1()),
  133. InnerVectorizedTraversal,CompleteUnrolling));
  134. VERIFY(test_assign(Vector1(),Vector1().template cast<Scalar>(),
  135. InnerVectorizedTraversal,CompleteUnrolling));
  136. VERIFY(test_assign(Matrix44(),Matrix44()+Matrix44(),
  137. InnerVectorizedTraversal,InnerUnrolling));
  138. VERIFY(test_assign(Matrix44u(),Matrix44()+Matrix44(),
  139. EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearTraversal,
  140. EIGEN_UNALIGNED_VECTORIZE ? InnerUnrolling : NoUnrolling));
  141. VERIFY(test_assign(Matrix1(),Matrix1()+Matrix1(),
  142. (int(Matrix1::InnerSizeAtCompileTime) % int(PacketSize))==0 ? InnerVectorizedTraversal : LinearVectorizedTraversal,
  143. CompleteUnrolling));
  144. VERIFY(test_assign(Matrix1u(),Matrix1()+Matrix1(),
  145. EIGEN_UNALIGNED_VECTORIZE ? ((int(Matrix1::InnerSizeAtCompileTime) % int(PacketSize))==0 ? InnerVectorizedTraversal : LinearVectorizedTraversal)
  146. : LinearTraversal, CompleteUnrolling));
  147. VERIFY(test_assign(Matrix44c().col(1),Matrix44c().col(2)+Matrix44c().col(3),
  148. InnerVectorizedTraversal,CompleteUnrolling));
  149. VERIFY(test_assign(Matrix44r().row(2),Matrix44r().row(1)+Matrix44r().row(1),
  150. InnerVectorizedTraversal,CompleteUnrolling));
  151. if(PacketSize>1)
  152. {
  153. typedef Matrix<Scalar,3,3,ColMajor> Matrix33c;
  154. typedef Matrix<Scalar,3,1,ColMajor> Vector3;
  155. VERIFY(test_assign(Matrix33c().row(2),Matrix33c().row(1)+Matrix33c().row(1),
  156. LinearTraversal,CompleteUnrolling));
  157. VERIFY(test_assign(Vector3(),Vector3()+Vector3(),
  158. sizeof(Scalar)==16 ? InnerVectorizedTraversal : (EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal), CompleteUnrolling));
  159. VERIFY(test_assign(Matrix33c().col(0),Matrix33c().col(1)+Matrix33c().col(1),
  160. EIGEN_UNALIGNED_VECTORIZE ? (sizeof(Scalar)==16 ? InnerVectorizedTraversal : LinearVectorizedTraversal)
  161. : (sizeof(Scalar)==16 ? SliceVectorizedTraversal : LinearTraversal),
  162. ((!EIGEN_UNALIGNED_VECTORIZE) && (sizeof(Scalar)==16)) ? NoUnrolling : CompleteUnrolling));
  163. VERIFY(test_assign(Matrix3(),Matrix3().cwiseProduct(Matrix3()),
  164. LinearVectorizedTraversal,CompleteUnrolling));
  165. VERIFY(test_assign(Matrix<Scalar,17,17>(),Matrix<Scalar,17,17>()+Matrix<Scalar,17,17>(),
  166. sizeof(Scalar)==16 ? InnerVectorizedTraversal :
  167. EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal :
  168. LinearTraversal,
  169. NoUnrolling));
  170. VERIFY(test_assign(Matrix11(), Matrix11()+Matrix11(),InnerVectorizedTraversal,CompleteUnrolling));
  171. VERIFY(test_assign(Matrix11(),Matrix<Scalar,21,21>().template block<PacketSize,PacketSize>(2,3)+Matrix<Scalar,21,21>().template block<PacketSize,PacketSize>(3,2),
  172. (EIGEN_UNALIGNED_VECTORIZE) ? InnerVectorizedTraversal : DefaultTraversal, CompleteUnrolling|InnerUnrolling));
  173. VERIFY(test_assign(Vector1(),Matrix11()*Vector1(),
  174. InnerVectorizedTraversal,CompleteUnrolling));
  175. VERIFY(test_assign(Matrix11(),Matrix11().lazyProduct(Matrix11()),
  176. InnerVectorizedTraversal,InnerUnrolling+CompleteUnrolling));
  177. }
  178. VERIFY(test_redux(Vector1(),
  179. LinearVectorizedTraversal,CompleteUnrolling));
  180. VERIFY(test_redux(Vector1().array()*Vector1().array(),
  181. LinearVectorizedTraversal,CompleteUnrolling));
  182. VERIFY(test_redux((Vector1().array()*Vector1().array()).col(0),
  183. LinearVectorizedTraversal,CompleteUnrolling));
  184. VERIFY(test_redux(Matrix<Scalar,PacketSize,3>(),
  185. LinearVectorizedTraversal,CompleteUnrolling));
  186. VERIFY(test_redux(Matrix3(),
  187. LinearVectorizedTraversal,CompleteUnrolling));
  188. VERIFY(test_redux(Matrix44(),
  189. LinearVectorizedTraversal,NoUnrolling));
  190. if(PacketSize>1) {
  191. VERIFY(test_redux(Matrix44().template block<(Matrix1::Flags&RowMajorBit)?4:PacketSize,(Matrix1::Flags&RowMajorBit)?PacketSize:4>(1,2),
  192. SliceVectorizedTraversal,CompleteUnrolling));
  193. VERIFY(test_redux(Matrix44().template block<(Matrix1::Flags&RowMajorBit)?2:PacketSize,(Matrix1::Flags&RowMajorBit)?PacketSize:2>(1,2),
  194. DefaultTraversal,CompleteUnrolling));
  195. }
  196. VERIFY(test_redux(Matrix44c().template block<2*PacketSize,1>(1,2),
  197. LinearVectorizedTraversal,CompleteUnrolling));
  198. VERIFY(test_redux(Matrix44r().template block<1,2*PacketSize>(2,1),
  199. LinearVectorizedTraversal,CompleteUnrolling));
  200. VERIFY((test_assign<
  201. Map<Matrix22, AlignedMax, OuterStride<3*PacketSize> >,
  202. Matrix22
  203. >(InnerVectorizedTraversal,CompleteUnrolling)));
  204. VERIFY((test_assign<
  205. Map<Matrix<Scalar,EIGEN_PLAIN_ENUM_MAX(2,PacketSize),EIGEN_PLAIN_ENUM_MAX(2,PacketSize)>, AlignedMax, InnerStride<3*PacketSize> >,
  206. Matrix<Scalar,EIGEN_PLAIN_ENUM_MAX(2,PacketSize),EIGEN_PLAIN_ENUM_MAX(2,PacketSize)>
  207. >(DefaultTraversal,PacketSize>=8?InnerUnrolling:CompleteUnrolling)));
  208. VERIFY((test_assign(Matrix11(), Matrix<Scalar,PacketSize,EIGEN_PLAIN_ENUM_MIN(2,PacketSize)>()*Matrix<Scalar,EIGEN_PLAIN_ENUM_MIN(2,PacketSize),PacketSize>(),
  209. InnerVectorizedTraversal, CompleteUnrolling)));
  210. #endif
  211. VERIFY(test_assign(MatrixXX(10,10),MatrixXX(20,20).block(10,10,2,3),
  212. SliceVectorizedTraversal,NoUnrolling));
  213. VERIFY(test_redux(VectorX(10),
  214. LinearVectorizedTraversal,NoUnrolling));
  215. }
  216. };
  217. template<typename Scalar> struct vectorization_logic<Scalar,false>
  218. {
  219. static void run() {}
  220. };
  221. template<typename Scalar, bool Enable = !internal::is_same<typename internal::unpacket_traits<typename internal::packet_traits<Scalar>::type>::half,
  222. typename internal::packet_traits<Scalar>::type>::value >
  223. struct vectorization_logic_half
  224. {
  225. typedef internal::packet_traits<Scalar> PacketTraits;
  226. typedef typename internal::unpacket_traits<typename internal::packet_traits<Scalar>::type>::half PacketType;
  227. enum {
  228. PacketSize = internal::unpacket_traits<PacketType>::size
  229. };
  230. static void run()
  231. {
  232. typedef Matrix<Scalar,PacketSize,1> Vector1;
  233. typedef Matrix<Scalar,PacketSize,PacketSize> Matrix11;
  234. typedef Matrix<Scalar,5*PacketSize,7,ColMajor> Matrix57;
  235. typedef Matrix<Scalar,3*PacketSize,5,ColMajor> Matrix35;
  236. typedef Matrix<Scalar,5*PacketSize,7,DontAlign|ColMajor> Matrix57u;
  237. typedef Matrix<Scalar,
  238. (PacketSize==16 ? 8 : PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1),
  239. (PacketSize==16 ? 2 : PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1)
  240. > Matrix1;
  241. typedef Matrix<Scalar,
  242. (PacketSize==16 ? 8 : PacketSize==8 ? 4 : PacketSize==4 ? 2 : PacketSize==2 ? 1 : /*PacketSize==1 ?*/ 1),
  243. (PacketSize==16 ? 2 : PacketSize==8 ? 2 : PacketSize==4 ? 2 : PacketSize==2 ? 2 : /*PacketSize==1 ?*/ 1),
  244. DontAlign|((Matrix1::Flags&RowMajorBit)?RowMajor:ColMajor)> Matrix1u;
  245. // this type is made such that it can only be vectorized when viewed as a linear 1D vector
  246. typedef Matrix<Scalar,
  247. (PacketSize==16 ? 4 : PacketSize==8 ? 4 : PacketSize==4 ? 6 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?2:3) : /*PacketSize==1 ?*/ 1),
  248. (PacketSize==16 ? 12 : PacketSize==8 ? 6 : PacketSize==4 ? 2 : PacketSize==2 ? ((Matrix11::Flags&RowMajorBit)?3:2) : /*PacketSize==1 ?*/ 3)
  249. > Matrix3;
  250. #if !EIGEN_GCC_AND_ARCH_DOESNT_WANT_STACK_ALIGNMENT
  251. VERIFY(test_assign(Vector1(),Vector1(),
  252. InnerVectorizedTraversal,CompleteUnrolling));
  253. VERIFY(test_assign(Vector1(),Vector1()+Vector1(),
  254. InnerVectorizedTraversal,CompleteUnrolling));
  255. VERIFY(test_assign(Vector1(),Vector1().template segment<PacketSize>(0).derived(),
  256. EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearVectorizedTraversal,CompleteUnrolling));
  257. VERIFY(test_assign(Vector1(),Scalar(2.1)*Vector1()-Vector1(),
  258. InnerVectorizedTraversal,CompleteUnrolling));
  259. VERIFY(test_assign(Vector1(),(Scalar(2.1)*Vector1().template segment<PacketSize>(0)-Vector1().template segment<PacketSize>(0)).derived(),
  260. EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearVectorizedTraversal,CompleteUnrolling));
  261. VERIFY(test_assign(Vector1(),Vector1().cwiseProduct(Vector1()),
  262. InnerVectorizedTraversal,CompleteUnrolling));
  263. VERIFY(test_assign(Vector1(),Vector1().template cast<Scalar>(),
  264. InnerVectorizedTraversal,CompleteUnrolling));
  265. VERIFY(test_assign(Matrix57(),Matrix57()+Matrix57(),
  266. InnerVectorizedTraversal,InnerUnrolling));
  267. VERIFY(test_assign(Matrix57u(),Matrix57()+Matrix57(),
  268. EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : LinearTraversal,
  269. EIGEN_UNALIGNED_VECTORIZE ? InnerUnrolling : NoUnrolling));
  270. VERIFY(test_assign(Matrix1u(),Matrix1()+Matrix1(),
  271. EIGEN_UNALIGNED_VECTORIZE ? ((int(Matrix1::InnerSizeAtCompileTime) % int(PacketSize))==0 ? InnerVectorizedTraversal : LinearVectorizedTraversal) : LinearTraversal,CompleteUnrolling));
  272. if(PacketSize>1)
  273. {
  274. typedef Matrix<Scalar,3,3,ColMajor> Matrix33c;
  275. VERIFY(test_assign(Matrix33c().row(2),Matrix33c().row(1)+Matrix33c().row(1),
  276. LinearTraversal,CompleteUnrolling));
  277. VERIFY(test_assign(Matrix33c().col(0),Matrix33c().col(1)+Matrix33c().col(1),
  278. EIGEN_UNALIGNED_VECTORIZE ? (sizeof(Scalar)==16 ? InnerVectorizedTraversal : LinearVectorizedTraversal)
  279. : (sizeof(Scalar)==16 ? SliceVectorizedTraversal : LinearTraversal),
  280. ((!EIGEN_UNALIGNED_VECTORIZE) && (sizeof(Scalar)==16)) ? NoUnrolling : CompleteUnrolling));
  281. VERIFY(test_assign(Matrix3(),Matrix3().cwiseQuotient(Matrix3()),
  282. PacketTraits::HasDiv ? LinearVectorizedTraversal : LinearTraversal,CompleteUnrolling));
  283. VERIFY(test_assign(Matrix<Scalar,17,17>(),Matrix<Scalar,17,17>()+Matrix<Scalar,17,17>(),
  284. sizeof(Scalar)==16 ? InnerVectorizedTraversal : (EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal),
  285. NoUnrolling));
  286. VERIFY(test_assign(Matrix11(),Matrix<Scalar,17,17>().template block<PacketSize,PacketSize>(2,3)+Matrix<Scalar,17,17>().template block<PacketSize,PacketSize>(8,4),
  287. EIGEN_UNALIGNED_VECTORIZE ? InnerVectorizedTraversal : DefaultTraversal,InnerUnrolling+CompleteUnrolling));
  288. VERIFY(test_assign(Vector1(),Matrix11()*Vector1(),
  289. InnerVectorizedTraversal,CompleteUnrolling));
  290. VERIFY(test_assign(Matrix11(),Matrix11().lazyProduct(Matrix11()),
  291. InnerVectorizedTraversal,InnerUnrolling+CompleteUnrolling));
  292. }
  293. VERIFY(test_redux(Vector1(),
  294. LinearVectorizedTraversal,CompleteUnrolling));
  295. VERIFY(test_redux(Matrix<Scalar,PacketSize,3>(),
  296. LinearVectorizedTraversal,CompleteUnrolling));
  297. VERIFY(test_redux(Matrix3(),
  298. LinearVectorizedTraversal,CompleteUnrolling));
  299. VERIFY(test_redux(Matrix35(),
  300. LinearVectorizedTraversal,CompleteUnrolling));
  301. VERIFY(test_redux(Matrix57().template block<PacketSize==1?2:PacketSize,3>(1,0),
  302. SliceVectorizedTraversal,CompleteUnrolling));
  303. if(PacketSize>1) {
  304. VERIFY(test_redux(Matrix57().template block<PacketSize,2>(1,0),
  305. DefaultTraversal,CompleteUnrolling));
  306. }
  307. VERIFY((test_assign<
  308. Map<Matrix<Scalar,EIGEN_PLAIN_ENUM_MAX(2,PacketSize),EIGEN_PLAIN_ENUM_MAX(2,PacketSize)>, AlignedMax, InnerStride<3*PacketSize> >,
  309. Matrix<Scalar,EIGEN_PLAIN_ENUM_MAX(2,PacketSize),EIGEN_PLAIN_ENUM_MAX(2,PacketSize)>
  310. >(DefaultTraversal,PacketSize>4?InnerUnrolling:CompleteUnrolling)));
  311. VERIFY((test_assign(Matrix57(), Matrix<Scalar,5*PacketSize,3>()*Matrix<Scalar,3,7>(),
  312. InnerVectorizedTraversal, InnerUnrolling+CompleteUnrolling)));
  313. #endif
  314. }
  315. };
  316. template<typename Scalar> struct vectorization_logic_half<Scalar,false>
  317. {
  318. static void run() {}
  319. };
  320. EIGEN_DECLARE_TEST(vectorization_logic)
  321. {
  322. #ifdef EIGEN_VECTORIZE
  323. CALL_SUBTEST( vectorization_logic<int>::run() );
  324. CALL_SUBTEST( vectorization_logic<float>::run() );
  325. CALL_SUBTEST( vectorization_logic<double>::run() );
  326. CALL_SUBTEST( vectorization_logic<std::complex<float> >::run() );
  327. CALL_SUBTEST( vectorization_logic<std::complex<double> >::run() );
  328. CALL_SUBTEST( vectorization_logic_half<int>::run() );
  329. CALL_SUBTEST( vectorization_logic_half<float>::run() );
  330. CALL_SUBTEST( vectorization_logic_half<double>::run() );
  331. CALL_SUBTEST( vectorization_logic_half<std::complex<float> >::run() );
  332. CALL_SUBTEST( vectorization_logic_half<std::complex<double> >::run() );
  333. if(internal::packet_traits<float>::Vectorizable)
  334. {
  335. VERIFY(test_assign(Matrix<float,3,3>(),Matrix<float,3,3>()+Matrix<float,3,3>(),
  336. EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal,CompleteUnrolling));
  337. VERIFY(test_redux(Matrix<float,5,2>(),
  338. EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : DefaultTraversal,CompleteUnrolling));
  339. }
  340. if(internal::packet_traits<double>::Vectorizable)
  341. {
  342. VERIFY(test_assign(Matrix<double,3,3>(),Matrix<double,3,3>()+Matrix<double,3,3>(),
  343. EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : LinearTraversal,CompleteUnrolling));
  344. VERIFY(test_redux(Matrix<double,7,3>(),
  345. EIGEN_UNALIGNED_VECTORIZE ? LinearVectorizedTraversal : DefaultTraversal,CompleteUnrolling));
  346. }
  347. #endif // EIGEN_VECTORIZE
  348. }