Rollup merge of #126530 - beetrees:f16-inline-asm-riscv, r=Amanieu · rust-lang/rust@e7956cd
@@ -13,7 +13,7 @@ use rustc_codegen_ssa::traits::*;
1313use rustc_data_structures::fx::FxHashMap;
1414use rustc_middle::ty::layout::TyAndLayout;
1515use rustc_middle::{bug, span_bug, ty::Instance};
16-use rustc_span::{Pos, Span};
16+use rustc_span::{sym, Pos, Span, Symbol};
1717use rustc_target::abi::*;
1818use rustc_target::asm::*;
1919use tracing::debug;
@@ -64,7 +64,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
6464let mut layout = None;
6565let ty = if let Some(ref place) = place {
6666 layout = Some(&place.layout);
67-llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
67+llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)
6868} else if matches!(
6969 reg.reg_class(),
7070InlineAsmRegClass::X86(
@@ -112,7 +112,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
112112// so we just use the type of the input.
113113&in_value.layout
114114};
115-let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout);
115+let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);
116116 output_types.push(ty);
117117 op_idx.insert(idx, constraints.len());
118118let prefix = if late { "=" } else { "=&" };
@@ -127,8 +127,13 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
127127for (idx, op) in operands.iter().enumerate() {
128128match *op {
129129InlineAsmOperandRef::In { reg, value } => {
130-let llval =
131-llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);
130+let llval = llvm_fixup_input(
131+self,
132+ value.immediate(),
133+ reg.reg_class(),
134+&value.layout,
135+ instance,
136+);
132137 inputs.push(llval);
133138 op_idx.insert(idx, constraints.len());
134139 constraints.push(reg_to_llvm(reg, Some(&value.layout)));
@@ -139,6 +144,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
139144 in_value.immediate(),
140145 reg.reg_class(),
141146&in_value.layout,
147+ instance,
142148);
143149 inputs.push(value);
144150@@ -341,7 +347,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
341347} else {
342348self.extract_value(result, op_idx[&idx] as u64)
343349};
344-let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout);
350+let value =
351+llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);
345352OperandValue::Immediate(value).store(self, place);
346353}
347354}
@@ -913,12 +920,22 @@ fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Ty
913920}
914921}
915922923+fn any_target_feature_enabled(
924+cx: &CodegenCx<'_, '_>,
925+instance: Instance<'_>,
926+features: &[Symbol],
927+) -> bool {
928+let enabled = cx.tcx.asm_target_features(instance.def_id());
929+ features.iter().any(|feat| enabled.contains(feat))
930+}
931+916932/// Fix up an input value to work around LLVM bugs.
917933fn llvm_fixup_input<'ll, 'tcx>(
918934bx: &mut Builder<'_, 'll, 'tcx>,
919935mut value: &'ll Value,
920936reg: InlineAsmRegClass,
921937layout: &TyAndLayout<'tcx>,
938+instance: Instance<'_>,
922939) -> &'ll Value {
923940let dl = &bx.tcx.data_layout;
924941match (reg, layout.abi) {
@@ -1029,6 +1046,16 @@ fn llvm_fixup_input<'ll, 'tcx>(
10291046 _ => value,
10301047}
10311048}
1049+(InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
1050+if s.primitive() == Primitive::Float(Float::F16)
1051+&& !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1052+{
1053+// Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.
1054+let value = bx.bitcast(value, bx.type_i16());
1055+let value = bx.zext(value, bx.type_i32());
1056+let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1057+ bx.bitcast(value, bx.type_f32())
1058+}
10321059 _ => value,
10331060}
10341061}
@@ -1039,6 +1066,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
10391066mut value: &'ll Value,
10401067reg: InlineAsmRegClass,
10411068layout: &TyAndLayout<'tcx>,
1069+instance: Instance<'_>,
10421070) -> &'ll Value {
10431071match (reg, layout.abi) {
10441072(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
@@ -1140,6 +1168,14 @@ fn llvm_fixup_output<'ll, 'tcx>(
11401168 _ => value,
11411169}
11421170}
1171+(InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
1172+if s.primitive() == Primitive::Float(Float::F16)
1173+&& !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1174+{
1175+let value = bx.bitcast(value, bx.type_i32());
1176+let value = bx.trunc(value, bx.type_i16());
1177+ bx.bitcast(value, bx.type_f16())
1178+}
11431179 _ => value,
11441180}
11451181}
@@ -1149,6 +1185,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
11491185cx: &CodegenCx<'ll, 'tcx>,
11501186reg: InlineAsmRegClass,
11511187layout: &TyAndLayout<'tcx>,
1188+instance: Instance<'_>,
11521189) -> &'ll Type {
11531190match (reg, layout.abi) {
11541191(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
@@ -1242,6 +1279,12 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
12421279 _ => layout.llvm_type(cx),
12431280}
12441281}
1282+(InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))
1283+if s.primitive() == Primitive::Float(Float::F16)
1284+&& !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
1285+{
1286+ cx.type_f32()
1287+}
12451288 _ => layout.llvm_type(cx),
12461289}
12471290}