
def to_valid_utf8(str)

  return str if str.valid_encoding?
    
  utf8_codepoint = Array.new
  new_str = ""

  # first reencode it
  reencoded = str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '?')

  # then check for valid byte sequence
  reencoded.each_byte do |byte|
    case byte
      
      # illegal bytes
      when 0, 192, 193, 245..255
        utf8_codepoint.clear

      # ASCII plane
      when 1..127
        if utf8_codepoint[0]
          utf8_codepoint.clear
        else
          new_str += [byte].pack('C*').force_encoding('UTF-8')
        end
        
      # start byte of multibyte sequence
      when 194..244
        if utf8_codepoint[0]
          utf8_codepoint.clear
        else
          utf8_codepoint << byte
        end

      # continuation byte
      when 128..191        
        # we need a valid start byte
        if utf8_codepoint.size == 0
          utf8_codepoint.clear
          next
        end
        
        case utf8_codepoint[0]
          
          # 2 byte sequence
          when 194..223       
            char = get_char_from_sequence(byte, utf8_codepoint, 2)
            new_str += char if char

          # 3 byte sequence
          when 224..239
            char = get_char_from_sequence(byte, utf8_codepoint, 3)
            new_str += char if char
            
          # 4 byte sequence
          when 240..244
            char = get_char_from_sequence(byte, utf8_codepoint, 4)
            new_str += char if char
          
          # this should not happen
          else
            utf8_codepoint.clear

        end
        
    end
  end

  return new_str
end

def get_char_from_sequence(byte, actual_codepoint, size)

  new_char = nil

  # check for illegal size
  if actual_codepoint.size > (size - 1) 
    actual_codepoint.clear

  else
    # character is valid
    actual_codepoint << byte

    if actual_codepoint.size == size
      new_char = actual_codepoint.pack('U*')
      actual_codepoint.clear
    end
  end
  
  return new_char
end