BitMagic-C++
|
Compute functions for SSE4.2 SIMD instruction set (internal) More...
#include <mmintrin.h>
#include <emmintrin.h>
#include <smmintrin.h>
#include <nmmintrin.h>
#include <immintrin.h>
#include "bmdef.h"
#include "bmsse_util.h"
#include "bmutil.h"
Go to the source code of this file.
Namespaces | |
bm | |
Macros | |
#define | VECT_XOR_ARR_2_MASK(dst, src, src_end, mask) sse2_xor_arr_2_mask((__m128i*)(dst), (__m128i*)(src), (__m128i*)(src_end), (bm::word_t)mask) |
#define | VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, mask) sse2_andnot_arr_2_mask((__m128i*)(dst), (__m128i*)(src), (__m128i*)(src_end), (bm::word_t)mask) |
#define | VECT_BITCOUNT(first, last) sse4_bit_count((__m128i*) (first), (__m128i*) (last)) |
#define | VECT_BITCOUNT_AND(first, last, mask) sse4_bit_count_op((__m128i*) (first), (__m128i*) (last), (__m128i*) (mask), sse2_and) |
#define | VECT_BITCOUNT_OR(first, last, mask) sse4_bit_count_op((__m128i*) (first), (__m128i*) (last), (__m128i*) (mask), sse2_or) |
#define | VECT_BITCOUNT_XOR(first, last, mask) sse4_bit_count_op((__m128i*) (first), (__m128i*) (last), (__m128i*) (mask), sse2_xor) |
#define | VECT_BITCOUNT_SUB(first, last, mask) sse4_bit_count_op((__m128i*) (first), (__m128i*) (last), (__m128i*) (mask), sse2_sub) |
#define | VECT_INVERT_BLOCK(first) sse2_invert_block((__m128i*)first); |
#define | VECT_AND_BLOCK(dst, src) sse4_and_block((__m128i*) dst, (__m128i*) (src)) |
#define | VECT_AND_DIGEST(dst, src) sse4_and_digest((__m128i*) dst, (const __m128i*) (src)) |
#define | VECT_AND_DIGEST_5WAY(dst, src1, src2, src3, src4) sse4_and_digest_5way((__m128i*) dst, (const __m128i*) (src1), (const __m128i*) (src2), (const __m128i*) (src3), (const __m128i*) (src4)) |
#define | VECT_AND_DIGEST_2WAY(dst, src1, src2) sse4_and_digest_2way((__m128i*) dst, (const __m128i*) (src1), (const __m128i*) (src2)) |
#define | VECT_OR_BLOCK(dst, src) sse2_or_block((__m128i*) dst, (__m128i*) (src)) |
#define | VECT_OR_BLOCK_2WAY(dst, src1, src2) sse2_or_block_2way((__m128i*) (dst), (const __m128i*) (src1), (const __m128i*) (src2)) |
#define | VECT_OR_BLOCK_3WAY(dst, src1, src2) sse2_or_block_3way((__m128i*) (dst), (const __m128i*) (src1), (const __m128i*) (src2)) |
#define | VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4) sse2_or_block_5way((__m128i*) (dst), (__m128i*) (src1), (__m128i*) (src2), (__m128i*) (src3), (__m128i*) (src4)) |
#define | VECT_SUB_BLOCK(dst, src) sse2_sub_block((__m128i*) dst, (const __m128i*) (src)) |
#define | VECT_SUB_DIGEST(dst, src) sse4_sub_digest((__m128i*) dst, (const __m128i*) (src)) |
#define | VECT_SUB_DIGEST_2WAY(dst, src1, src2) sse4_sub_digest_2way((__m128i*) dst, (const __m128i*) (src1), (const __m128i*) (src2)) |
#define | VECT_XOR_BLOCK(dst, src) sse2_xor_block((__m128i*) dst, (__m128i*) (src)) |
#define | VECT_XOR_BLOCK_2WAY(dst, src1, src2) sse2_xor_block_2way((__m128i*) (dst), (const __m128i*) (src1), (const __m128i*) (src2)) |
#define | VECT_COPY_BLOCK(dst, src) sse2_copy_block((__m128i*) dst, (__m128i*) (src)) |
#define | VECT_STREAM_BLOCK(dst, src) sse2_stream_block((__m128i*) dst, (__m128i*) (src)) |
#define | VECT_SET_BLOCK(dst, value) sse2_set_block((__m128i*) dst, value) |
#define | VECT_IS_ZERO_BLOCK(dst) sse4_is_all_zero((__m128i*) dst) |
#define | VECT_IS_ONE_BLOCK(dst) sse4_is_all_one((__m128i*) dst) |
#define | VECT_IS_DIGEST_ZERO(start) sse4_is_digest_zero((__m128i*)start) |
#define | VECT_BLOCK_SET_DIGEST(dst, val) sse4_block_set_digest((__m128i*)dst, val) |
#define | VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to) sse4_lower_bound_scan_u32(arr, target, from, to) |
#define | VECT_SHIFT_L1(b, acc, co) sse42_shift_l1((__m128i*)b, acc, co) |
#define | VECT_SHIFT_R1(b, acc, co) sse42_shift_r1((__m128i*)b, acc, co) |
#define | VECT_SHIFT_R1_AND(b, co, m, digest) sse42_shift_r1_and((__m128i*)b, co, (__m128i*)m, digest) |
#define | VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start) sse42_idx_arr_block_lookup(idx, size, nb, start) |
#define | VECT_SET_BLOCK_BITS(block, idx, start, stop) sse42_set_block_bits(block, idx, start, stop) |
#define | VECT_BLOCK_CHANGE(block) sse42_bit_block_calc_change((__m128i*)block) |
Functions | |
bm::id_t | bm::sse4_bit_count (const __m128i *block, const __m128i *block_end) |
BMFORCEINLINE unsigned | bm::op_xor (unsigned a, unsigned b) |
BMFORCEINLINE unsigned | bm::op_or (unsigned a, unsigned b) |
BMFORCEINLINE unsigned | bm::op_and (unsigned a, unsigned b) |
template<class Func > | |
bm::id_t | bm::sse4_bit_count_op (const __m128i *BMRESTRICT block, const __m128i *BMRESTRICT block_end, const __m128i *BMRESTRICT mask_block, Func sse2_func) |
bool | bm::sse4_is_all_zero (const __m128i *BMRESTRICT block) |
check if block is all zero bits More... | |
bool | bm::sse4_is_digest_zero (const __m128i *BMRESTRICT block) |
check if digest stride is all zero bits More... | |
void | bm::sse4_block_set_digest (__m128i *dst, unsigned value) |
set digest stride to 0xFF.. or 0x0 value More... | |
unsigned | bm::sse4_and_block (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) |
AND blocks2 dst &= *src. More... | |
bool | bm::sse4_and_digest (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) |
AND block digest stride dst &= *src. More... | |
bool | bm::sse4_and_digest_2way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) |
AND block digest stride dst = *src1 & src2. More... | |
bool | bm::sse4_and_digest_5way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2, const __m128i *BMRESTRICT src3, const __m128i *BMRESTRICT src4) |
AND block digest stride. More... | |
bool | bm::sse4_sub_digest (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src) |
SUB (AND NOT) block digest stride dst &= ~*src. More... | |
bool | bm::sse4_sub_digest_2way (__m128i *BMRESTRICT dst, const __m128i *BMRESTRICT src1, const __m128i *BMRESTRICT src2) |
2-operand SUB (AND NOT) block digest stride dst = src1 & ~*src2 More... | |
bool | bm::sse4_is_all_one (const __m128i *BMRESTRICT block) |
check if block is all zero bits More... | |
BMFORCEINLINE bool | bm::sse42_test_all_zero_wave (const void *ptr) |
check if wave of pointers is all NULL More... | |
BMFORCEINLINE bool | bm::sse42_test_all_zero_wave2 (const void *ptr0, const void *ptr1) |
check if 2 waves of pointers are all NULL More... | |
BMFORCEINLINE bool | bm::sse42_test_all_eq_wave2 (const void *ptr0, const void *ptr1) |
check if wave of 2 pointers are the same (null or FULL) More... | |
unsigned | bm::sse42_bit_block_calc_change (const __m128i *BMRESTRICT block) |
unsigned | bm::sse4_gap_find (const bm::gap_word_t *BMRESTRICT pbuf, const bm::gap_word_t pos, const unsigned size) |
int | bm::sse42_cmpge_u32 (__m128i vect4, unsigned value) |
Experimental (test) function to do SIMD vector search (lower bound) in sorted, growing array. More... | |
unsigned | bm::sse4_lower_bound_scan_u32 (const unsigned *BMRESTRICT arr, unsigned target, unsigned from, unsigned to) |
lower bound (great or equal) linear scan in ascending order sorted array More... | |
unsigned | bm::sse42_idx_arr_block_lookup (const unsigned *idx, unsigned size, unsigned nb, unsigned start) |
void | bm::sse42_set_block_bits (bm::word_t *BMRESTRICT block, const unsigned *BMRESTRICT idx, unsigned start, unsigned stop) |
void | bm::sse4_bit_block_gather_scatter (unsigned *BMRESTRICT arr, const unsigned *BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx) |
bool | bm::sse42_shift_l1 (__m128i *block, unsigned *empty_acc, unsigned co1) |
block shift left by 1 More... | |
bool | bm::sse42_shift_r1 (__m128i *block, unsigned *empty_acc, unsigned co1) |
block shift right by 1 More... | |
bool | bm::sse42_shift_r1_and (__m128i *block, bm::word_t co1, const __m128i *BMRESTRICT mask_block, bm::id64_t *digest) |
block shift right by 1 plus AND More... | |
Compute functions for SSE4.2 SIMD instruction set (internal)
Definition in file bmsse4.h.
#define VECT_AND_BLOCK | ( | dst, | |
src | |||
) | sse4_and_block((__m128i*) dst, (__m128i*) (src)) |
Definition at line 1309 of file bmsse4.h.
Referenced by bm::bit_block_and().
#define VECT_AND_DIGEST | ( | dst, | |
src | |||
) | sse4_and_digest((__m128i*) dst, (const __m128i*) (src)) |
Definition at line 1312 of file bmsse4.h.
Referenced by bm::bit_block_and().
#define VECT_AND_DIGEST_2WAY | ( | dst, | |
src1, | |||
src2 | |||
) | sse4_and_digest_2way((__m128i*) dst, (const __m128i*) (src1), (const __m128i*) (src2)) |
Definition at line 1318 of file bmsse4.h.
Referenced by bm::bit_block_and_2way().
#define VECT_AND_DIGEST_5WAY | ( | dst, | |
src1, | |||
src2, | |||
src3, | |||
src4 | |||
) | sse4_and_digest_5way((__m128i*) dst, (const __m128i*) (src1), (const __m128i*) (src2), (const __m128i*) (src3), (const __m128i*) (src4)) |
Definition at line 1315 of file bmsse4.h.
Referenced by bm::bit_block_and_5way().
#define VECT_ANDNOT_ARR_2_MASK | ( | dst, | |
src, | |||
src_end, | |||
mask | |||
) | sse2_andnot_arr_2_mask((__m128i*)(dst), (__m128i*)(src), (__m128i*)(src_end), (bm::word_t)mask) |
Definition at line 1288 of file bmsse4.h.
Referenced by bm::bit_andnot_arr_ffmask().
#define VECT_ARR_BLOCK_LOOKUP | ( | idx, | |
size, | |||
nb, | |||
start | |||
) | sse42_idx_arr_block_lookup(idx, size, nb, start) |
Definition at line 1381 of file bmsse4.h.
Referenced by bm::idx_arr_block_lookup_u32().
#define VECT_BITCOUNT | ( | first, | |
last | |||
) | sse4_bit_count((__m128i*) (first), (__m128i*) (last)) |
Definition at line 1291 of file bmsse4.h.
Referenced by bm::bit_block_count().
#define VECT_BITCOUNT_AND | ( | first, | |
last, | |||
mask | |||
) | sse4_bit_count_op((__m128i*) (first), (__m128i*) (last), (__m128i*) (mask), sse2_and) |
Definition at line 1294 of file bmsse4.h.
Referenced by bm::bit_block_and_count().
#define VECT_BITCOUNT_OR | ( | first, | |
last, | |||
mask | |||
) | sse4_bit_count_op((__m128i*) (first), (__m128i*) (last), (__m128i*) (mask), sse2_or) |
Definition at line 1297 of file bmsse4.h.
Referenced by bm::bit_block_or_count().
#define VECT_BITCOUNT_SUB | ( | first, | |
last, | |||
mask | |||
) | sse4_bit_count_op((__m128i*) (first), (__m128i*) (last), (__m128i*) (mask), sse2_sub) |
Definition at line 1303 of file bmsse4.h.
Referenced by bm::bit_block_sub_count().
#define VECT_BITCOUNT_XOR | ( | first, | |
last, | |||
mask | |||
) | sse4_bit_count_op((__m128i*) (first), (__m128i*) (last), (__m128i*) (mask), sse2_xor) |
Definition at line 1300 of file bmsse4.h.
Referenced by bm::bit_block_xor_count().
#define VECT_BLOCK_CHANGE | ( | block | ) | sse42_bit_block_calc_change((__m128i*)block) |
Definition at line 1387 of file bmsse4.h.
Referenced by bm::bit_block_calc_change().
#define VECT_BLOCK_SET_DIGEST | ( | dst, | |
val | |||
) | sse4_block_set_digest((__m128i*)dst, val) |
Definition at line 1366 of file bmsse4.h.
Referenced by bm::block_init_digest0().
#define VECT_COPY_BLOCK | ( | dst, | |
src | |||
) | sse2_copy_block((__m128i*) dst, (__m128i*) (src)) |
Definition at line 1348 of file bmsse4.h.
Referenced by bm::bit_block_copy().
#define VECT_INVERT_BLOCK | ( | first | ) | sse2_invert_block((__m128i*)first); |
Definition at line 1306 of file bmsse4.h.
Referenced by bm::bit_invert().
#define VECT_IS_DIGEST_ZERO | ( | start | ) | sse4_is_digest_zero((__m128i*)start) |
Definition at line 1363 of file bmsse4.h.
Referenced by bm::calc_block_digest0(), and bm::update_block_digest0().
#define VECT_IS_ONE_BLOCK | ( | dst | ) | sse4_is_all_one((__m128i*) dst) |
Definition at line 1360 of file bmsse4.h.
Referenced by bm::is_bits_one().
#define VECT_IS_ZERO_BLOCK | ( | dst | ) | sse4_is_all_zero((__m128i*) dst) |
Definition at line 1357 of file bmsse4.h.
Referenced by bm::bit_is_all_zero().
#define VECT_LOWER_BOUND_SCAN_U32 | ( | arr, | |
target, | |||
from, | |||
to | |||
) | sse4_lower_bound_scan_u32(arr, target, from, to) |
Definition at line 1369 of file bmsse4.h.
Referenced by bm::lower_bound_linear_u32().
#define VECT_OR_BLOCK | ( | dst, | |
src | |||
) | sse2_or_block((__m128i*) dst, (__m128i*) (src)) |
Definition at line 1321 of file bmsse4.h.
Referenced by bm::bit_block_or().
#define VECT_OR_BLOCK_2WAY | ( | dst, | |
src1, | |||
src2 | |||
) | sse2_or_block_2way((__m128i*) (dst), (const __m128i*) (src1), (const __m128i*) (src2)) |
Definition at line 1324 of file bmsse4.h.
Referenced by bm::bit_block_or_2way().
#define VECT_OR_BLOCK_3WAY | ( | dst, | |
src1, | |||
src2 | |||
) | sse2_or_block_3way((__m128i*) (dst), (const __m128i*) (src1), (const __m128i*) (src2)) |
Definition at line 1327 of file bmsse4.h.
Referenced by bm::bit_block_or_3way().
#define VECT_OR_BLOCK_5WAY | ( | dst, | |
src1, | |||
src2, | |||
src3, | |||
src4 | |||
) | sse2_or_block_5way((__m128i*) (dst), (__m128i*) (src1), (__m128i*) (src2), (__m128i*) (src3), (__m128i*) (src4)) |
Definition at line 1330 of file bmsse4.h.
Referenced by bm::bit_block_or_5way().
#define VECT_SET_BLOCK | ( | dst, | |
value | |||
) | sse2_set_block((__m128i*) dst, value) |
Definition at line 1354 of file bmsse4.h.
Referenced by bm::bit_block_set().
#define VECT_SET_BLOCK_BITS | ( | block, | |
idx, | |||
start, | |||
stop | |||
) | sse42_set_block_bits(block, idx, start, stop) |
Definition at line 1384 of file bmsse4.h.
Referenced by bm::set_block_bits_u32().
#define VECT_SHIFT_L1 | ( | b, | |
acc, | |||
co | |||
) | sse42_shift_l1((__m128i*)b, acc, co) |
Definition at line 1372 of file bmsse4.h.
Referenced by bm::bit_block_shift_l1_unr().
#define VECT_SHIFT_R1 | ( | b, | |
acc, | |||
co | |||
) | sse42_shift_r1((__m128i*)b, acc, co) |
Definition at line 1375 of file bmsse4.h.
Referenced by bm::bit_block_shift_r1_unr().
#define VECT_SHIFT_R1_AND | ( | b, | |
co, | |||
m, | |||
digest | |||
) | sse42_shift_r1_and((__m128i*)b, co, (__m128i*)m, digest) |
Definition at line 1378 of file bmsse4.h.
Referenced by bm::bit_block_shift_r1_and_unr().
#define VECT_STREAM_BLOCK | ( | dst, | |
src | |||
) | sse2_stream_block((__m128i*) dst, (__m128i*) (src)) |
Definition at line 1351 of file bmsse4.h.
Referenced by bm::bit_block_stream().
#define VECT_SUB_BLOCK | ( | dst, | |
src | |||
) | sse2_sub_block((__m128i*) dst, (const __m128i*) (src)) |
Definition at line 1333 of file bmsse4.h.
Referenced by bm::bit_block_sub().
#define VECT_SUB_DIGEST | ( | dst, | |
src | |||
) | sse4_sub_digest((__m128i*) dst, (const __m128i*) (src)) |
Definition at line 1336 of file bmsse4.h.
Referenced by bm::bit_block_sub().
#define VECT_SUB_DIGEST_2WAY | ( | dst, | |
src1, | |||
src2 | |||
) | sse4_sub_digest_2way((__m128i*) dst, (const __m128i*) (src1), (const __m128i*) (src2)) |
Definition at line 1339 of file bmsse4.h.
Referenced by bm::bit_block_sub_2way().
#define VECT_XOR_ARR_2_MASK | ( | dst, | |
src, | |||
src_end, | |||
mask | |||
) | sse2_xor_arr_2_mask((__m128i*)(dst), (__m128i*)(src), (__m128i*)(src_end), (bm::word_t)mask) |
Definition at line 1285 of file bmsse4.h.
Referenced by bm::bvector<>::combine_operation().
#define VECT_XOR_BLOCK | ( | dst, | |
src | |||
) | sse2_xor_block((__m128i*) dst, (__m128i*) (src)) |
Definition at line 1342 of file bmsse4.h.
Referenced by bm::bit_block_xor().
#define VECT_XOR_BLOCK_2WAY | ( | dst, | |
src1, | |||
src2 | |||
) | sse2_xor_block_2way((__m128i*) (dst), (const __m128i*) (src1), (const __m128i*) (src2)) |
Definition at line 1345 of file bmsse4.h.
Referenced by bm::bit_block_xor_2way().