UnsafeMutablePointer в AURenderCallback

В Swift 3 инициализаторы нельзя использовать для преобразования типов указателей. В вашем случае тип inRefCon равен UnsafeMutableRawPointer, поэтому вам нужно использовать метод assumingMemoryBound(to:).

И еще одно, адрес player, переданный в callback, должен быть стабильным все время воспроизведения звука, адреса, взятые из аргументов inout (указанные префиксом &), не удовлетворяют этому требованию.

Две вещи выше исправлены, ваш код будет примерно таким:

import Foundation
import AudioToolbox

let sineFrequency = 880.0

// MARK: User data struct
struct SineWavePlayer {
    var outputUnit: AudioUnit? = nil
    var startingFrameCount: Double = 0
}

// MARK: Callback function
let SineWaveRenderProc: AURenderCallback = {(inRefCon, ioActionFlags, inTimeStamp, inBusNumber, inNumberFrames, ioData) -> OSStatus in

    var player = inRefCon.assumingMemoryBound(to: SineWavePlayer.self)


    var j = player.pointee.startingFrameCount
    let cycleLength = 44100 / sineFrequency

    for frame in 0..<inNumberFrames {
        var buffers = UnsafeMutableAudioBufferListPointer(ioData)

        buffers?[0].mData?.assumingMemoryBound(to: Float32.self)[Int(frame)] = Float32(sin(2 * M_PI * (j / cycleLength)))
        buffers?[1].mData?.assumingMemoryBound(to: Float32.self)[Int(frame)] = Float32(sin(2 * M_PI * (j / cycleLength)))

        j += 1
        if j > cycleLength {
            j -= cycleLength
        }
    }

    player.pointee.startingFrameCount = j
    return noErr
}

// MARK: Utility function
func CheckError(_ error: OSStatus, operation: String) {
    guard error != noErr else {
        return
    }

    var result: String = ""
    var char = Int(error.bigEndian)

    for _ in 0..<4 {
        guard isprint(Int32(char&255)) == 1 else {
            result = "\(error)"
            break
        }
        result.append(String(describing: UnicodeScalar(char&255)))
        char = char/256
    }

    print("Error: \(operation) (\(result))")

    exit(1)
}

func CreateAndConnectOutputUnit(_ playerPtr: UnsafeMutablePointer<SineWavePlayer>) {
    // Generate a description that matches the output device (speakers)
    var outputcd = AudioComponentDescription(componentType: kAudioUnitType_Output, componentSubType: kAudioUnitSubType_DefaultOutput, componentManufacturer: kAudioUnitManufacturer_Apple, componentFlags: 0, componentFlagsMask: 0)

    let comp = AudioComponentFindNext(nil, &outputcd)

    if comp == nil {
        print("Can't get output unit")
        exit(-1)
    }

    CheckError(AudioComponentInstanceNew(comp!, &playerPtr.pointee.outputUnit),
               operation: "Couldn't open component for outputUnit")

    // Register the render callback
    var input = AURenderCallbackStruct(inputProc: SineWaveRenderProc, inputProcRefCon: playerPtr)

    CheckError(AudioUnitSetProperty(playerPtr.pointee.outputUnit!, kAudioUnitProperty_SetRenderCallback, kAudioUnitScope_Input, 0, &input, UInt32(MemoryLayout<AURenderCallbackStruct>.size)),
               operation: "AudioUnitSetProperty failed")

    // Initialize the unit
    CheckError(AudioUnitInitialize(playerPtr.pointee.outputUnit!),
               operation: "Couldn't initialize output unit")
}

func main() {
    let playerPtr = UnsafeMutablePointer<SineWavePlayer>.allocate(capacity: 1)
    defer {playerPtr.deallocate(capacity: 1)}
    playerPtr.initialize(to: SineWavePlayer())
    defer {playerPtr.deinitialize()}

    // Set up output unit and callback
    CreateAndConnectOutputUnit(playerPtr)

    // Start playing
    CheckError(AudioOutputUnitStart(playerPtr.pointee.outputUnit!),
               operation: "Couldn't start output unit")

    // Play for 5 seconds
    sleep(5)

    // Clean up
    AudioOutputUnitStop(playerPtr.pointee.outputUnit!)
    AudioUnitUninitialize(playerPtr.pointee.outputUnit!)
    AudioComponentInstanceDispose(playerPtr.pointee.outputUnit!)
}